#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Datetime: 2021/10/30 15:09
# @Author  : CHEN Wang
# @Site    : 
# @File    : trading_data_weeks.py
# @Software: PyCharm

"""
脚本说明: 计算与价量相关衍生指标, 每三天计算一次
"""

import os
import numpy as np
import pandas as pd
from datetime import datetime, timedelta
from quant_researcher.quant.project_tool.time_tool import get_yesterday, date_shifter, get_the_end_of_this_month, calc_date_diff
from quant_researcher.quant.datasource_fetch.crypto_api.glassnode import get_prices
from quant_researcher.quant.project_tool.localize import DATA_DIR
from TQR_Applications.factor_database.crypto.trading_data_update import get_all_ohlcvm_data
from quant_researcher.quant.datasource_fetch.crypto_api.coinmarketcap import get_all_coinlist_via_http
from quant_researcher.quant.project_tool.file_tool import get_all_filename_path
from quant_researcher.quant.datasource_fetch.crypto_api.self_defined import all_stalecoin_crypto
from quant_researcher.quant.performance_attribution.core_functions.performance_analysis.performance import prices_cleaning
from quant_researcher.quant.project_tool import hammer
from quant_researcher.quant.factors.factor_preprocess.preprocess import ts_threshold_signal
from sklearn.decomposition import PCA
from logger import logger
import os
import traceback
from task_monitor import task_to_db, send_error_to_email

pd.options.mode.chained_assignment = None  # default='warn'


def ret_cleaning_process(split_df, **kwargs):
    """

    :param split_df: 需要处理的收益率数据, index为日期，columns为币名
    :param kwargs:
        - all_amount_df: 用来获取每个收益率前60的交易额情况
        - total_amount_series： 用来确定最小交易额阈值
        - ico_date_series： ico后7天交易额不做限制
    :return:
    """

    all_amount_df = kwargs.pop('all_amount_df')
    total_amount_series = kwargs.pop('total_amount_series')
    ico_date_series = kwargs.pop('ico_date_series')

    # 把所有需要进行小额交易过滤的收益率转换成三列，一列为日期，一列为收益率，一列为币名称
    all_ret_series_list = []
    for i in split_df.columns:
        temp_ret_series = split_df[i]
        temp_ret_series.dropna(inplace=True)
        temp_ret_series.name = 'ret'
        temp_temp_ret_df = pd.DataFrame(temp_ret_series)
        temp_temp_ret_df['name'] = i
        all_ret_series_list.append(temp_temp_ret_df)
    temp_all_ret_df = pd.concat(all_ret_series_list, axis=0)
    # temp_all_ret_df.sort_values(by=['ret'], inplace=True, ascending=False)  # 不做排序的话，则按照时间顺序处理
    temp_all_ret_df.reset_index(inplace=True)

    # 通过简单粗暴的收益率阈值进行过滤
    # if ret_type == 'daily':  # 日收益率对应的异常收益率
    #     max_ret_threshold = 0.5
    #     min_ret_threshold = -0.5
    # elif ret_type == 'weekly':   # 周收益率对应的异常收益率
    #     max_ret_threshold = 1
    #     min_ret_threshold = -0.6
    # elif ret_type == 'monthly':  # 月收益率对应的异常收益率
    #     max_ret_threshold = 2
    #     min_ret_threshold = -0.8
    # temp_all_ret_df = temp_all_ret_df[(temp_all_ret_df['ret'] > max_ret_threshold) | (temp_all_ret_df['ret'] < min_ret_threshold)]

    # '2017-09-01'至 '2017-10-01'期间因为交易所清退，交易额可能存在异常情况，直接忽略这一段收益率，不进行过滤
    kick_out_date = pd.date_range('2017-09-01', '2017-10-01', freq='D').tolist()
    kick_out_date = [x.strftime("%Y-%m-%d") for x in kick_out_date]

    latest_date = all_amount_df.index[-1]

    # # 需要人为剔除的异常涨跌幅 todo 有些需要人为筛选剔除， 查看 All_Coins_{ret_type}_return_series_smalltrade_Flase_after_filter
    # human_kick_coins = [['2021-10-28', 'dogezilla'],
    #                     ['2021-04-08', 'xrpdown'],
    #                     ['2018-03-09', 'pac-protocol'],
    #                     ['2020-10-27', 'certik'],
    #                     ['2021-05-28', 'ethereummax'],
    #                     ['2022-01-11', 'eth-fan-token'],
    #                     ['2021-03-17', 'nestree'],
    #                     ['2016-03-28', 'bios-crypto'],
    #                     ['2022-04-19', 'wonderhero'],
    #                     ['2022-03-28', 'bxmi-token'],
    #                     ['2016-07-07', 'invisiblecoin'],
    #                     ['2021-05-04', 'klayswap-protocol'],
    #                     ['2022-03-19', 'titi-financial'],
    #                     ['2017-03-27', 'bitcoin-plus'],
    #                     ['2021-10-28', 'squid-game'],
    #                     ['2021-08-14', 'folgory-coin'],
    #                     ['2021-12-18', 'eminer'],
    #                     ['2021-03-19', 'wanbtc'],
    #                     ['2017-10-29', 'akuya-coin']]

    # 收益率进行小额交易过滤
    all_ret_df = split_df.copy()
    for index, this_row in temp_all_ret_df.iterrows():  # 逐个异常收益率进行分析
        this_date = this_row['end_date']  # 该收益率对应的日期
        next_60_day = date_shifter(this_date, step='days', how_many=60)  # 后60天对应日期
        previous_60_day = date_shifter(this_date, step='days', how_many=-60)  # 前60天对应日期
        this_name = this_row['name']  # 这个币的名称
        ico_date = ico_date_series[this_name]  # 这个币的ICO日期
        ico_next_7_days = date_shifter(ico_date, step='days', how_many=7)  # ICO的后7天

        if previous_60_day < ico_next_7_days:  # 如果前60天超过了该币的的ICO7天后，则小额交易过滤起始日期为ICO7天后
            previous_60_day = ico_next_7_days  # ico后7天交易额不做限制
        # if next_60_day > latest_date:  # 如果后60天日期超过了所有数据的最新日期，则小额交易过滤结束日期为最新日期
        #     next_60_day = latest_date  # 交易额限制只截止到我们统计时间段的最后一天
        # this_coin_recent_amount = all_amount_df.loc[previous_60_day:next_60_day, this_name]  # todo 该收益率前后60天收益率均需要最小交易额要求，否则认定为异常收益率
        this_coin_recent_amount = all_amount_df.loc[previous_60_day:this_date, this_name]  # todo 该收益率前60天收益率包括当天需要最小交易额要求,后60天不做要求，否则认定为异常收益率

        all_date_list = list(this_coin_recent_amount.index)
        all_cal_date_list = [i for i in all_date_list if i not in kick_out_date]  # 部分日期不做交易额要求
        this_coin_recent_amount = this_coin_recent_amount[all_cal_date_list]
        small_amount_threshold = total_amount_series[all_cal_date_list] / 100000
        small_amount_threshold[small_amount_threshold < 20000] = 20000  # 计算出来最小交易额阈值小于20000的统统设置为20000

        if not ((this_coin_recent_amount - small_amount_threshold) >= 0).all():  # 如果当天交易额以及前后60天交易额有一个不满足最小交易额要求，这个收益率认为是异常收益率
            all_ret_df.loc[this_date, this_name] = np.nan  # 异常收益率直接填充为NaN
            temp_all_ret_df.loc[index, 'filter_or_not'] = True
        # elif [this_date, this_name] in human_kick_coins:  # 暂不需要人为过滤
        #     all_ret_df.loc[this_date, this_name] = np.nan
        #     temp_all_ret_df.loc[index, 'filter_or_not'] = True
        else:
            pass

    del this_coin_recent_amount
    del small_amount_threshold
    del split_df

    return all_ret_df, temp_all_ret_df


def special_max_drawdown_series(prices):
    """
    计算每时每刻的最大回撤数据; 以2020-03-12为分割点；因为有些币2021年的高点不如2017年高

    :param pd.Series prices: 价格序列
    :return:
    """

    idx = prices.index
    hwm = pd.Series(0.0, index=idx)

    hwm[idx[0]] = prices[idx[0]]
    # Create the high water mark
    for i, t in enumerate(idx[1:]):
        # hwm[t] = max(hwm[idx[i]], prices[t])
        if t != '2020-03-12':
            if str(hwm[idx[i]]) == 'nan':  # 上一期的高点数据为空
                hwm[t] = prices[t]
            else:
                hwm[t] = max(hwm[idx[i]], prices[t])
        elif t == '2020-03-12':  # 以该时间节点分割，前后的最大回撤分别计算
            hwm[t] = prices[t]  # 高点重新记
        else:
            raise NotImplementedError

    hwm.replace(0, np.nan, inplace=True)
    max_drawdown_series = (prices - hwm) / hwm
    max_drawdown_series.name = prices.name
    return max_drawdown_series


def each_coin_data_format(coins_info, **kwargs):
    i = coins_info[0]
    coins = coins_info[1]
    all_filename_path_dict = kwargs.pop('all_filename_path_dict', None)

    print(i)
    file_name = all_filename_path_dict[f'{coins}.xlsx']
    history_data = pd.read_excel(f'{file_name}', index_col='end_date')
    if history_data.empty:
        print(f'{coins}.xlsx中的数据全为空')
        return None, None, None
    history_data.sort_values(by=['datetime'], inplace=True)
    history_data = history_data[~history_data.index.duplicated(keep='last')]  # 同一天多条记录进行去重
    history_close = history_data['close']
    history_amount = history_data['amount']
    history_marketcap = history_data['market_cap']
    history_close.name = coins
    history_amount.name = coins
    history_marketcap.name = coins
    return history_close, history_amount, history_marketcap


def data_reformat():
    # 统计各币的涨跌幅情况 - 根据coinmarketcap的数据
    file_path = os.path.join(DATA_DIR, r'all_history_ohlcvm_coinmarketcap\all_history_ohlcvm_coinmarketcap')
    all_filename_path_dict = get_all_filename_path(file_path)
    all_coins_name = [i.replace('.xlsx', '') for i in all_filename_path_dict.keys()]

    # # 批处理框架，但是joblib最大的问题在于结束后不会自动关闭进程
    # from joblib import Parallel, delayed
    # result = Parallel(n_jobs=60, verbose=1)(delayed(each_coin_data_format)([i, coins], all_filename_path_dict) for i, coins in enumerate(all_coins_name))

    # 批处理框架
    info = {
        'func': each_coin_data_format,
        'all_filename_path_dict': all_filename_path_dict
    }
    list_for_test = [[i, coins] for i, coins in enumerate(all_coins_name)]
    result = hammer.split_a_list_into_batches_and_deal_each_with_a_process(
        the_list=list_for_test, info=info, ignore_err=True, how_many=50, what_version='v2')
    all_prices_tuple, all_amount_tuple, all_marketcap_tuple = zip(*result)
    all_prices_df = pd.concat(list(all_prices_tuple), axis=1)
    all_prices_df.sort_index(inplace=True)
    all_amount_df = pd.concat(list(all_amount_tuple), axis=1)
    all_amount_df.sort_index(inplace=True)
    all_marketcap_df = pd.concat(list(all_marketcap_tuple), axis=1)
    all_marketcap_df.sort_index(inplace=True)

    # # 文件保存本地
    # file_path = os.path.join(DATA_DIR, r'all_history_ohlcvm_coinmarketcap')
    # file_name = os.path.join(file_path, f'all_history_ohlcvm_prices')
    # all_prices_df.to_excel(f'{file_name}.xlsx')
    # file_name = os.path.join(file_path, f'all_history_ohlcvm_amount')
    # all_amount_df.to_excel(f'{file_name}.xlsx')
    # file_name = os.path.join(file_path, f'all_history_ohlcvm_marketcap')
    # all_marketcap_df.to_excel(f'{file_name}.xlsx')

    return all_prices_df, all_amount_df, all_marketcap_df


def get_analysis_coin_list(all_prices_df):
    # 剔除各种不需要分析的币，得到最终需要分析的币的名单

    # 剔除稳定币
    all_coins_name = list(all_prices_df.columns)
    # stablecoin_names = [i for i in all_coins_name if i in list(all_stalecoin_crypto.values())]
    file_path_1 = os.path.join(DATA_DIR, 'all_history_ohlcvm_coinmarketcap')
    file_name_1 = os.path.join(file_path_1, f'all_stablecoin_list_coinmarketcap')
    all_stablecoin_list_df = pd.read_excel(f'{file_name_1}.xlsx', index_col='slug')
    stablecoin_names = list(all_stablecoin_list_df.index)
    all_coins_name_without_stablecoins = [i for i in all_coins_name if i not in stablecoin_names]

    # 有些币是wrapped，其实走势是一致的，需要剔除掉
    # 剔除掉带wrapped-前缀的
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if 'wrapped-' not in i]
    # 剔除掉带staked的
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if 'staked' not in i]
    # 发布在不同链上的同一代币
    differentchain_similar_coin_list = ['bitcoin-bep2', 'sheesha-finance-bep20',
                                        'sheesha-finance-polygon', 'astronaut-polygon', 'atlantis-loans-polygon', 'mm-finance-polygon', 'mm-optimizer-polygon',
                                        'phenix-finance-polygon', 'sheesha-finance-polygon', 'waultswap-polygon', 'wolf-safe-poor-people-polygon']
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if i not in differentchain_similar_coin_list]

    # 换了名字的币
    differentname_coin_list = ['certick']  # certik改名为shentu
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if i not in differentname_coin_list]

    # 剔除掉一些特殊的wrapped的币
    similar_coin_list = ['algomint-gobtc', 'bitcoin-additional', 'bitcoin-avalanche-bridged', 'btc-proxy', 'huobi-btc', 'interbtc', 'kintsugi-btc', 'oec-btc', 'renbtc', 'rsk-smart-bitcoin', 'sbtc', 'tbtc',
                         'wbnb', 'stader-bnbx', 'stkbnb',
                         'steth', 'aave-ethereum', 'beacon-eth', 'ethk', 'goeth', 'lido-finance-wsteth', 'peth', 'rocket-pool-eth', 'seth', 'seth2', 'stakewise-reth2', 'waneth', 'weth', 'ankreth',
                         'lido-staked-matic', 'stader-maticx', 'wmatic', 'xmatic', 'ankr-avalanche-reward-earning-bond', 'ankr-reward-earning-matic',
                         'usdj', 'aave-busd', 'aave-dai', 'axlusdc', 'balancer-boosted-aave-usdc', 'curve-fi-frax-usdc', 'curve-fi-gdai-gusdc-gusdt', 'orbit-bridge-klaytn-usdc', 'usd-coin-avalanche-bridged-usdc-e', 'wxdai',
                         'inverse-bitcoin-volatility-token', 'crypto-volatility-token']
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if i not in similar_coin_list]
    # 剔除掉一些杠杆代币
    leverage_coin_list = ['btcdown', 'dotdown', 'eth-down', 'ethdown', 'linkdown', 'trxdown', 'xrpdown', 'adadown', 'bnbdown',
                          'btcup', 'dotup', 'eth-up', 'ethup', 'linkup', 'trxup', 'xrpup', 'adaup', 'bnbup']
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if i not in leverage_coin_list]
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-short-' not in i]
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-long-' not in i]
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-leverage-' not in i]
    # 剔除一些指数代币
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-index' not in i]
    # 剔除一些股票代币
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-stock-' not in i]
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-tokenized-' not in i]

    # # 保存all_coins_name_without_stablecoins，看看剔除后还剩哪些币
    # all_coins_name_without_stablecoins_series = pd.Series(index=all_coins_name_without_stablecoins)
    # file_name_1 = os.path.join(file_path_1, f'all_coins_name_without_stablecoins_series')
    # all_coins_name_without_stablecoins_series.to_excel(f'{file_name_1}.xlsx')

    return all_coins_name_without_stablecoins


def get_trading_data(asset, start_date, end_date=None):
    """
    计算与价量相关衍生指标

    :param str asset: 资产代码
    :param str start_date: '2014-01-01'
    :param str end_date: '2014-01-01'
    :return:
    """

    if end_date is None:
        end_date = get_yesterday(marker='with_n_dash')  # 计算截止昨日收盘

    price_log_price = get_prices(ohlc=False, asset=asset, start_date=start_date, end_date=end_date)
    price_log_price['log_prices'] = np.log10(price_log_price['close'])

    # 把文件夹中各个小文件中的价格，市值，交易额数据汇总成一个文件
    # 如果之前已经读取过，不用重复读取，读写这些数据 all_prices_df, all_amount_df, all_marketcap_df
    file_path = os.path.join(DATA_DIR, 'all_history_ohlcvm_coinmarketcap')
    temp_file_name = os.path.join(file_path, f'all_history_ohlcvm_marketcap_raw')
    history_all_marketcap_df = pd.read_csv(f'{temp_file_name}.csv', index_col='end_date')
    history_date = history_all_marketcap_df.index[-1]
    temp_file_path = os.path.join(DATA_DIR, r'all_history_ohlcvm_coinmarketcap\all_history_ohlcvm_coinmarketcap')
    temp_file_name = os.path.join(temp_file_path, f'bitcoin')
    history_bitcoin_df = pd.read_excel(f'{temp_file_name}.xlsx', index_col='end_date')
    bitcoin_history_date = history_bitcoin_df.index[-1]
    if calc_date_diff(history_date, bitcoin_history_date) == 0:  # 说明all_prices_df， all_amount_df， all_marketcap_df已经读取汇总并保存过了，无需再保存
        file_name = os.path.join(file_path, f'all_history_ohlcvm_prices_raw')
        all_prices_df = pd.read_csv(f'{file_name}.csv', index_col='end_date')
        file_name = os.path.join(file_path, f'all_history_ohlcvm_amount_raw')
        all_amount_df = pd.read_csv(f'{file_name}.csv', index_col='end_date')
        file_name = os.path.join(file_path, f'all_history_ohlcvm_marketcap_raw')
        all_marketcap_df = pd.read_csv(f'{file_name}.csv', index_col='end_date')
    else:
        print('开始加载全量历史数据-prices, amount, marketcap')
        all_prices_df, all_amount_df, all_marketcap_df = data_reformat()
        file_name = os.path.join(file_path, f'all_history_ohlcvm_prices_raw')
        all_prices_df.to_csv(f'{file_name}.csv')
        file_name = os.path.join(file_path, f'all_history_ohlcvm_amount_raw')
        all_amount_df.to_csv(f'{file_name}.csv')
        file_name = os.path.join(file_path, f'all_history_ohlcvm_marketcap_raw')
        all_marketcap_df.to_csv(f'{file_name}.csv')

    all_prices_df = all_prices_df.astype(np.float32)
    print('对价格数据做清洗')
    temp_file_name = os.path.join(file_path, f'all_history_ohlcvm_price_cleaned')
    if not os.path.exists(f'{temp_file_name}.csv'):
        all_prices_df = all_prices_df.apply(prices_cleaning)
        all_prices_df.dropna(how='all', axis=1, inplace=True)
        all_prices_df.to_csv(f'{temp_file_name}.csv')
    else:
        history_all_cleaned_price_df = pd.read_csv(f'{temp_file_name}.csv', index_col='end_date')
        # history_all_cleaned_price_df.set_index('end_date', inplace=True)
        history_date_temp = history_all_cleaned_price_df.index[-1]
        if calc_date_diff(history_date_temp, bitcoin_history_date) == 0:  # 说明all_prices_df已经清洗过了，直接读取
            all_prices_df = history_all_cleaned_price_df
        else:
            all_prices_df = all_prices_df.apply(prices_cleaning)
            all_prices_df.dropna(how='all', axis=1, inplace=True)
            all_prices_df.to_csv(f'{temp_file_name}.csv')
    print('对价格数据清洗完成')

    # todo 统计各个币都是什么时候开始有行情的，可以类似于ICO的时间？？
    ico_date_series = all_prices_df.apply(lambda x: x[~x.isnull()].index[0])

    # 全市场总的交易金额, 用于后续计算每天最小交易额要求
    file_path_1 = os.path.join(DATA_DIR, 'trading_data')
    file_name_1 = os.path.join(file_path_1, f'cex_total_spot_amount_coinmarketcap')
    total_amount_series = pd.read_excel(f'{file_name_1}.xlsx', index_col='end_date')['amount']
    total_amount_series = total_amount_series.rolling(30).mean()  # 每天全市场交易额的30日均线
    # total_amount_series = all_amount_df.sum(axis=1)  # 各个币交易额的加总，这里会重复计算
    # total_amount_series = all_amount_df['bitcoin']  # 以BTC交易额为准，作为最小交易额基准

    # 剔除各种不需要分析的币，得到最终需要分析的币的名单
    all_coins_name_without_stablecoins = get_analysis_coin_list(all_prices_df)

    all_prices_df = all_prices_df[all_coins_name_without_stablecoins]
    all_amount_df = all_amount_df[all_coins_name_without_stablecoins]
    all_marketcap_df = all_marketcap_df[all_coins_name_without_stablecoins]

    # 计算各个币的均线上穿情况
    all_bias300_df = all_prices_df - all_prices_df.rolling(300).mean()
    all_bias200_df = all_prices_df - all_prices_df.rolling(200).mean()
    all_bias155_df = all_prices_df - all_prices_df.rolling(155).mean()
    all_bias120_df = all_prices_df - all_prices_df.rolling(120).mean()
    all_bias60_df = all_prices_df - all_prices_df.rolling(60).mean()
    all_bias30_df = all_prices_df - all_prices_df.rolling(30).mean()

    all_osc300_df = (all_prices_df - all_prices_df.rolling(300).mean()) / all_prices_df.rolling(300).mean()
    all_osc200_df = (all_prices_df - all_prices_df.rolling(200).mean()) / all_prices_df.rolling(200).mean()
    all_osc155_df = (all_prices_df - all_prices_df.rolling(155).mean()) / all_prices_df.rolling(155).mean()
    all_osc120_df = (all_prices_df - all_prices_df.rolling(120).mean()) / all_prices_df.rolling(120).mean()
    all_osc60_df = (all_prices_df - all_prices_df.rolling(60).mean()) / all_prices_df.rolling(60).mean()
    all_osc30_df = (all_prices_df - all_prices_df.rolling(30).mean()) / all_prices_df.rolling(30).mean()

    # 计算近1, 3 , 4, 6月涨跌幅
    all_3monthly_ret_df = all_prices_df.pct_change(90)
    all_3monthly_ret_df = all_3monthly_ret_df.replace([float('inf'), float('-inf')], [np.nan, np.nan])  # 剔除异常涨跌幅
    all_3monthly_ret_df[all_3monthly_ret_df > 1000000] = np.nan
    all_4monthly_ret_df = all_prices_df.pct_change(120)
    all_4monthly_ret_df = all_4monthly_ret_df.replace([float('inf'), float('-inf')], [np.nan, np.nan])  # 剔除异常涨跌幅
    all_4monthly_ret_df[all_4monthly_ret_df > 1000000] = np.nan
    all_6monthly_ret_df = all_prices_df.pct_change(180)
    all_6monthly_ret_df = all_6monthly_ret_df.replace([float('inf'), float('-inf')], [np.nan, np.nan])  # 剔除异常涨跌幅
    all_6monthly_ret_df[all_6monthly_ret_df > 1000000] = np.nan

    # 创近1,3,6个月新高或新低
    all_high30_df = (all_prices_df - all_prices_df.rolling(30).max())
    all_high60_df = (all_prices_df - all_prices_df.rolling(60).max())
    all_high90_df = (all_prices_df - all_prices_df.rolling(90).max())
    all_high120_df = (all_prices_df - all_prices_df.rolling(120).max())
    all_high180_df = (all_prices_df - all_prices_df.rolling(180).max())
    all_low30_df = (all_prices_df.rolling(30).min() - all_prices_df)
    all_low60_df = (all_prices_df.rolling(60).min() - all_prices_df)
    all_low90_df = (all_prices_df.rolling(90).min() - all_prices_df)
    all_low120_df = (all_prices_df.rolling(129).min() - all_prices_df)
    all_low180_df = (all_prices_df.rolling(180).min() - all_prices_df)

    # 距离,1周，近1，3,4，6个月高点回撤，低点上涨幅度
    all_high7ret_df = (all_prices_df - all_prices_df.rolling(7).max()) / all_prices_df.rolling(7).max()
    all_high30ret_df = (all_prices_df - all_prices_df.rolling(30).max()) / all_prices_df.rolling(30).max()
    all_high90ret_df = (all_prices_df - all_prices_df.rolling(90).max()) / all_prices_df.rolling(90).max()
    all_high120ret_df = (all_prices_df - all_prices_df.rolling(120).max()) / all_prices_df.rolling(120).max()
    all_high180ret_df = (all_prices_df - all_prices_df.rolling(180).max()) / all_prices_df.rolling(180).max()
    all_low7ret_df = (all_prices_df - all_prices_df.rolling(7).min()) / all_prices_df.rolling(7).min()
    all_low30ret_df = (all_prices_df - all_prices_df.rolling(30).min()) / all_prices_df.rolling(30).min()
    all_low90ret_df = (all_prices_df - all_prices_df.rolling(90).min()) / all_prices_df.rolling(90).min()
    all_low120ret_df = (all_prices_df - all_prices_df.rolling(120).min()) / all_prices_df.rolling(120).min()
    all_low180ret_df = (all_prices_df - all_prices_df.rolling(180).min()) / all_prices_df.rolling(180).min()

    # # kdj, rsi等技术指标
    # import talib
    # all_rsi30_df = all_prices_df.apply(talib.RSI, timeperiod=30)
    # all_rsi90_df = all_prices_df.apply(talib.RSI, timeperiod=90)
    # all_rsi180_df = all_prices_df.apply(talib.RSI, timeperiod=180)
    # all_rsi_signal_df_list = []
    # for rsi_period in [30, 90, 180]:
    #     if rsi_period == 30:
    #         temp_rsi_df = all_rsi30_df
    #     elif rsi_period == 90:
    #         temp_rsi_df = all_rsi90_df
    #     elif rsi_period == 180:
    #         temp_rsi_df = all_rsi180_df
    #     for bins in [[20, 80], [30, 70], [40, 60]]:
    #         temp_file_name = os.path.join(file_path, f'all_rsi{rsi_period}_{bins}_signal_df')
    #         if os.path.exists(f'{temp_file_name}.csv'):
    #             signal_df = pd.read_csv(f'{temp_file_name}.csv', index_col='end_date')
    #         else:
    #             info = {'func': ts_threshold_signal, 'bins': bins, 'left_right': 'right', 'positive': True}
    #             list_for_test = [temp_rsi_df[coins] for coins in temp_rsi_df.columns]
    #             results = hammer.split_a_list_into_batches_and_deal_each_with_a_process(the_list=list_for_test, info=info, ignore_err=True, how_many=60, what_version='v2')
    #             signal_df = pd.concat(results, axis=1)
    #             signal_df = signal_df[temp_rsi_df.columns]
    #             signal_df.sort_index(inplace=True)
    #             signal_df.to_csv(f'{temp_file_name}.csv')
    #         all_rsi_signal_df_list.append(signal_df)

    # 截取需要分析的时间段,因为可能数据不是同一天获取的，分析截止日设置为最后一期币数大于8000个的日期【如果该日币数少于8000，则不分析】
    coins_num_eachday = all_prices_df.count(axis=1)
    all_days = list(coins_num_eachday.index)
    all_days.sort(reverse=True)
    for day in all_days:
        if coins_num_eachday[day] > 8000:
            analysis_end_date = day
            break
        else:
            continue
    all_prices_df = all_prices_df.loc[start_date:analysis_end_date, ]
    all_amount_df = all_amount_df.loc[start_date:analysis_end_date, ]
    all_marketcap_df = all_marketcap_df.loc[start_date:analysis_end_date, ]
    total_amount_series = total_amount_series[start_date:analysis_end_date]
    # all_prices_df， all_amount_df， all_marketcap_df保存
    temp_file_path = os.path.join(DATA_DIR, 'all_history_ohlcvm_coinmarketcap')
    if history_date == analysis_end_date:  # 说明all_prices_df， all_amount_df， all_marketcap_df已经保存过了，无需再保存
        pass
    else:
        temp_file_name = os.path.join(temp_file_path, f'all_history_ohlcvm_prices')
        all_prices_df.to_csv(f'{temp_file_name}.csv')
        temp_file_name = os.path.join(temp_file_path, f'all_history_ohlcvm_amount')
        all_amount_df.to_csv(f'{temp_file_name}.csv')
        temp_file_name = os.path.join(temp_file_path, f'all_history_ohlcvm_marketcap')
        all_marketcap_df.to_csv(f'{temp_file_name}.csv')
    del history_all_marketcap_df

    # 计算最大回撤数据
    # all_maxdrawdown_df = all_prices_df.apply(max_drawdown)  # 太耗时了
    # 批处理框架, 试试速度快不快
    print('计算最大回撤')
    temp_file_name = os.path.join(file_path, f'all_history_ohlcvm_price_maxdrawdown')
    if not os.path.exists(f'{temp_file_name}.csv'):
        info = {'func': special_max_drawdown_series}
        list_for_test = [all_prices_df[coins] for coins in all_prices_df.columns]
        results = hammer.split_a_list_into_batches_and_deal_each_with_a_process(
            the_list=list_for_test, info=info, ignore_err=True, how_many=30, what_version='v2')
        all_maxdrawdown_df = pd.concat(results, axis=1)
        all_maxdrawdown_df = all_maxdrawdown_df[all_prices_df.columns]
        all_maxdrawdown_df.to_csv(f'{temp_file_name}.csv')
    else:
        history_all_maxdrawdown_df = pd.read_csv(f'{temp_file_name}.csv', index_col='end_date')
        history_date_temp = history_all_maxdrawdown_df.index[-1]
        if calc_date_diff(history_date_temp, bitcoin_history_date) == 0:  # 说明最大回撤已经计算过了，直接读取
            all_maxdrawdown_df = history_all_maxdrawdown_df
        else:
            info = {'func': special_max_drawdown_series}
            list_for_test = [all_prices_df[coins] for coins in all_prices_df.columns]
            results = hammer.split_a_list_into_batches_and_deal_each_with_a_process(
                the_list=list_for_test, info=info, ignore_err=True, how_many=30, what_version='v2')
            all_maxdrawdown_df = pd.concat(results, axis=1)
            all_maxdrawdown_df = all_maxdrawdown_df[all_prices_df.columns]
            all_maxdrawdown_df.to_csv(f'{temp_file_name}.csv')

    # all_daily_ret_df = all_prices_df.apply(prices_to_returns, log_returns=False, freq='D')
    # all_weekly_ret_df = all_prices_df.apply(prices_to_returns, log_returns=False, freq='W')
    # all_monthly_ret_df = all_prices_df.apply(prices_to_returns, log_returns=False, freq='M')

    all_daily_ret_df = all_prices_df.pct_change()
    all_daily_ret_df = all_daily_ret_df.replace([float('inf'), float('-inf')], [np.nan, np.nan])  # 剔除异常涨跌幅
    all_daily_ret_df[all_daily_ret_df > 100] = np.nan
    all_weekly_ret_df = all_prices_df.pct_change(7)
    all_weekly_ret_df = all_weekly_ret_df.replace([float('inf'), float('-inf')], [np.nan, np.nan])  # 剔除异常涨跌幅
    all_weekly_ret_df[all_weekly_ret_df > 1000] = np.nan
    all_monthly_ret_df = all_prices_df.pct_change(30)
    all_monthly_ret_df = all_monthly_ret_df.replace([float('inf'), float('-inf')], [np.nan, np.nan])  # 剔除异常涨跌幅
    all_monthly_ret_df[all_monthly_ret_df > 10000] = np.nan

    print('对日收益，周收益，月收益分别分析')
    for ret_type, ret in enumerate([all_daily_ret_df, all_weekly_ret_df, all_monthly_ret_df]):
        all_ret_df = ret.dropna(how='all', axis=0)  # 剔除全为空的行, 计算收益率导致的前几行为空
        all_result_df = pd.DataFrame(index=all_ret_df.index)
        earliest_day = all_ret_df.index[0]
        for kick_small_amount in [True, False]:  # 这几列数据需要填充为list， 需要提前设置dtype为object
            for topcap_num in ['all', 'top100', 'top200']:
                all_result_df[f'{topcap_num}_all_cleaned_coins_name_{kick_small_amount}'] = np.nan
                all_result_df[f'{topcap_num}_all_cleaned_coins_name_{kick_small_amount}'] = all_result_df[f'{topcap_num}_all_cleaned_coins_name_{kick_small_amount}'].astype('object')

        if ret_type == 0:
            ret_type = 'daily'
            ret_gap = [float('-inf'), -1, -0.9, -0.8, -0.7, -0.6, -0.5, -0.4, -0.3, -0.2, -0.15, -0.1, -0.05, 0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.5, 2, 3, 5, 10, float('inf')]
        elif ret_type == 1:
            ret_type = 'weekly'
            ret_gap = [float('-inf'), -1, -0.9, -0.8, -0.7, -0.6, -0.5, -0.4, -0.3, -0.2, -0.15, -0.1, -0.05, 0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.5, 2, 3, 5, 10, float('inf')]
        elif ret_type == 2:
            ret_type = 'monthly'
            ret_gap = [float('-inf'), -1, -0.9, -0.8, -0.7, -0.6, -0.5, -0.4, -0.3, -0.2, -0.15, -0.1, -0.05, 0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.5, 2, 3, 5, 10, float('inf')]
        else:
            raise NotImplementedError

        ret_gap_list = [[ret_gap[i], ret_gap[i + 1]] for i in range(len(ret_gap) - 1)]

        # 统计整体的收益率分布情况, 统计剔除小的交易量
        for kick_small_amount in [True, False]:
            if kick_small_amount:
                small_amount_series = total_amount_series / 100000  # 最小交易额限制为当天全市场交易额除以100000
                small_amount_series[small_amount_series < 20000] = 20000  # 计算出来最小交易额阈值小于20000的统统设置为20000
                temp_all_amount_df_list = []
                for columns in all_amount_df.columns:
                    temp_series = all_amount_df[columns] - small_amount_series
                    temp_series.name = columns
                    temp_all_amount_df_list.append(temp_series)
                temp_all_amount_df = pd.concat(temp_all_amount_df_list, axis=1)
                temp_all_amount_df = temp_all_amount_df.loc[all_ret_df.index, :]  # all_ret_df的前几行为空被drop掉了，因此index的长度不一致
                temp_ret_df = all_ret_df[temp_all_amount_df >= 0]  # 所有满足最小交易额的收益率数据
            else:
                temp_ret_df = all_ret_df[all_amount_df >= 0]  # 如果无需剔除最小交易额，则只要求交易额数据大于等于0，其实就是要求不为空

            # 对BTC收益率进行统计
            btc_ret_series = temp_ret_df['bitcoin']
            btc_ret_series.dropna(inplace=True)
            btc_ret_describe = btc_ret_series.describe()
            btc_ret_describe['median'] = btc_ret_series.median()
            print(f'btc_{ret_type}_ret_describe_smalltrade_{kick_small_amount}')
            print(btc_ret_describe)
            btc_ret_series.name = 'ret'
            btc_ret_df = pd.DataFrame(btc_ret_series)
            btc_ret_df['name'] = 'bitcoin'
            btc_ret_df.sort_values(by=['ret'], inplace=True, ascending=False)
            temp_file_path = os.path.join(DATA_DIR, r'all_history_ohlcvm_coinmarketcap')
            temp_file_name = os.path.join(temp_file_path, f'BTC_{ret_type}_return_series_smalltrade_{kick_small_amount}')
            btc_ret_df.to_csv(f'{temp_file_name}.csv')

            # 所有收益率拼成一个series，好对所有收益率进行画图和统计
            all_ret_series_list = []
            for i in all_coins_name_without_stablecoins:
                temp_ret_series = temp_ret_df[i]
                temp_ret_series.dropna(inplace=True)
                temp_ret_series.name = 'ret'
                temp_temp_ret_df = pd.DataFrame(temp_ret_series)
                temp_temp_ret_df['name'] = i
                all_ret_series_list.append(temp_temp_ret_df)
            temp_all_ret_df = pd.concat(all_ret_series_list, axis=0)
            all_ret_series = temp_all_ret_df['ret']
            all_ret_describe = all_ret_series.describe()
            all_ret_describe['median'] = all_ret_series.median()
            print(f'all_{ret_type}_ret_describe_smalltrade_{kick_small_amount}')
            print(all_ret_describe)

            temp_all_ret_df.sort_values(by=['ret'], inplace=True, ascending=False)
            temp_file_path = os.path.join(DATA_DIR, r'all_history_ohlcvm_coinmarketcap')
            temp_file_name = os.path.join(temp_file_path, f'All_Coins_{ret_type}_return_series_smalltrade_{kick_small_amount}')
            temp_all_ret_df.to_csv(f'{temp_file_name}.csv')

            # all_ret_series.dropna(inplace=True)
            # all_ret_series[all_ret_series > 3] = 3                               # 去极值, 不然画出来的图很奇怪
            # # all_ret_series[all_ret_series < -1] = -1                             # 去极值, 不然画出来的图很奇怪； 按道理不存在收益率小于-1的，需要检查数据是否有误
            # # all_ret_series = winsorize(all_ret_series, methods='three_sigma')  # 去极值, 不然画出来的图很奇怪
            # # 注意直方图上添加核密度图,必须将直方图频数更改为频率,即normed参数设置成True
            # all_ret_series.plot(kind="hist", bins=100, color="steelblue", edgecolor="black", label="直方图")  # 直方图
            # # all_ret_series.plot(kind="kde", color="red", label="核密度图")  # 加核密度图
            # plt.xlabel("收益率")
            # plt.ylabel("频率")
            # plt.title(f"{ret_type}收益率分布-剔除小额交易{kick_small_amount}")
            # plt.legend()
            # plt.show()

            del all_ret_series
            del temp_all_ret_df

        temp_file_name = os.path.join(temp_file_path, f'all_history_ohlcvm_{ret_type}_ret_cleaned')
        if os.path.exists(f'{temp_file_name}.csv'):
            history_cleaned_all_ret_df = pd.read_csv(f'{temp_file_name}.csv', index_col='end_date')
            cleaned_date = history_cleaned_all_ret_df.index[-1]
            date_to_clean = date_shifter(before=cleaned_date, how_many=-3, step='days')
        else:
            cleaned_date = '2015-01-01'
            date_to_clean = '2015-01-01'

        if cleaned_date == all_ret_df.index[-1]:  # 说明收益率已经被清洗过了，不用再清洗了，直接读取
            all_ret_df = history_cleaned_all_ret_df.copy()
            del history_cleaned_all_ret_df
        else:
            # 收益率数据进行清洗，todo 周收益率和月收益率这么清洗可能不一定对
            # 所有收益率拼成一个series，收益率从高到低排序，对于特别高收益率和特别低的收益率进行清洗
            temp_ret_df = all_ret_df[all_amount_df >= 0]
            temp_ret_df = temp_ret_df.loc[date_to_clean:, :]
            # temp_ret_df = pd.DataFrame(all_ret_df['elastos'])
            # temp_ret_df.dropna(inplace=True)
            temp_ret_df.reset_index(inplace=True)
            temp_ret_df_list_for_test = []
            split_num = 0
            while split_num < len(temp_ret_df.index):  # 对收益率数据按日期进行切割（10天切一刀），方便多进程处理
                if split_num + 10 > len(temp_ret_df.index):
                    end_num = len(temp_ret_df.index)
                else:
                    end_num = split_num + 10
                split_temp_ret_df = temp_ret_df.iloc[split_num:end_num, :].copy()
                split_temp_ret_df.set_index('end_date', inplace=True)
                temp_ret_df_list_for_test.append(split_temp_ret_df)
                split_num = split_num + 10

            # 批处理框架
            info = {
                'func': ret_cleaning_process,
                'all_amount_df': all_amount_df,
                'total_amount_series': total_amount_series,
                'ico_date_series': ico_date_series,
            }

            split_cleaned_ret_df_list = hammer.split_a_list_into_batches_and_deal_each_with_a_process(
                the_list=temp_ret_df_list_for_test, info=info, ignore_err=True, how_many=40, what_version='v2')
            all_cleaned_ret_df_list = []
            all_cleaned_ret_series_list = []
            for split_cleaned_data_list in split_cleaned_ret_df_list:
                all_cleaned_ret_df_list.append(split_cleaned_data_list[0])
                all_cleaned_ret_series_list.append(split_cleaned_data_list[1])
            all_ret_df = pd.concat(all_cleaned_ret_df_list, axis=0)  # 清理好的all_ret_df
            temp_all_ret_df = pd.concat(all_cleaned_ret_series_list, axis=0)  # 清理好的all_ret_df对应的series格式
            # 清除内存
            del all_cleaned_ret_df_list
            del all_cleaned_ret_series_list
            del split_cleaned_ret_df_list

            if cleaned_date != '2015-01-01':  # 说明最近只清洗了部分
                all_ret_df = pd.concat([history_cleaned_all_ret_df, all_ret_df], axis=0)
                all_ret_df = all_ret_df[~all_ret_df.index.duplicated(keep='last')]

                temp_file_name = os.path.join(temp_file_path, f'All_Coins_{ret_type}_return_series_smalltrade_Flase_after_filter')
                historical_temp_all_ret_df = pd.read_csv(f'{temp_file_name}.csv', index_col='Unnamed: 0')
                temp_all_ret_df = pd.concat([historical_temp_all_ret_df, temp_all_ret_df], axis=0, ignore_index=True)
                temp_all_ret_df.drop_duplicates(subset=['end_date', 'name'], inplace=True)

            temp_file_path = os.path.join(DATA_DIR, r'all_history_ohlcvm_coinmarketcap')
            all_ret_df.sort_index(inplace=True)
            temp_file_name = os.path.join(temp_file_path, f'all_history_ohlcvm_{ret_type}_ret_cleaned')
            all_ret_df.to_csv(f'{temp_file_name}.csv')

            temp_all_ret_df.sort_values(by=['ret'], inplace=True, ascending=False)
            temp_all_ret_df = temp_all_ret_df[(temp_all_ret_df['ret'] > 0.5) | (temp_all_ret_df['ret'] < -0.5)]  # 只保存一部分，不然数据太长了，Excel支持不了
            temp_file_name = os.path.join(temp_file_path, f'All_Coins_{ret_type}_return_series_smalltrade_Flase_after_filter')
            temp_all_ret_df.to_csv(f'{temp_file_name}.csv')
            del temp_all_ret_df

        all_coins_name_df = pd.DataFrame(index=all_ret_df.index)
        for date in all_ret_df.index:
            print(f'开始计算{date}的数据')

            # 当天有价格数据的币的名称
            temp_price_series = all_prices_df.loc[date, :]
            temp_price_series = temp_price_series.dropna()
            all_coins_name_df.loc[date, f'all_coins_name'] = f'{list(temp_price_series.index)}'  # 所有符合条件的币的名字
            # 当天每个币的交易额情况
            amount_series = all_amount_df.loc[date, all_coins_name_without_stablecoins]
            amount_series = amount_series.replace(0, np.nan)  # 交易额为0的需要剔除

            # 剔除交易额过小的情况
            # for kick_small_amount in [True, False]:
            for kick_small_amount in [True]:
                if kick_small_amount:
                    small_amount_threshold_this_date = total_amount_series[date] / 100000
                    if small_amount_threshold_this_date < 20000:
                        small_amount_threshold_this_date = 20000
                else:
                    small_amount_threshold_this_date = 0
                temp_amount_series = amount_series[(amount_series >= small_amount_threshold_this_date)]  # 剔除交易额小于多少的币 todo 交易额为空的也剔除掉

                if temp_amount_series.empty:  # 当天没有交易额超过small_amount_threshold_this_date的币， 跳过该交易日
                    continue
                temp_ret_series = all_ret_df.loc[date, temp_amount_series.index]  # 剔除交易额大于多少的币的这一天收益率
                temp_cap_series = all_marketcap_df.loc[date, temp_amount_series.index]  # 剔除交易额大于多少的币的这一天市值
                temp_cap_series = temp_cap_series.replace(0, np.nan)  # 市值为0的需要剔除
                all_coins_this_date = set(temp_ret_series.dropna().index).intersection(set(temp_cap_series.dropna().index))  # todo 当天交易额满足要求，且收益率数据和市值数据不为空
                temp_amount_series = temp_amount_series[all_coins_this_date]
                if temp_amount_series.empty:  # 同时满足三个条件的一个币都没有，跳过该交易日
                    continue
                temp_ret_series = temp_ret_series[all_coins_this_date]
                temp_cap_series = temp_cap_series[all_coins_this_date]
                temp_ret_series.sort_values(ascending=False, inplace=True)  # 收益率从大到小排序

                # 币走势一致性（可以分析以下两个指标 1、通过PCA第一主成分贡献率；2、或者通过两两相关性大于某一阈值占比）时间窗口测试30， 60， 90， 120， 180， 365
                for previous_day in [10, 20, 30, 60, 90, 180, 365]:
                    days_ago = date_shifter(date, step='days', how_many=-previous_day)
                    if days_ago < earliest_day:  # 数据长度不够，跳过
                        pass
                    else:
                        # 两两相关性大于某一阈值占比
                        temp_ret_df = all_ret_df.loc[days_ago: date, all_coins_this_date]
                        corr_df = temp_ret_df.corr(method='pearson', min_periods=10)
                        corr_up_df = pd.DataFrame(np.triu(corr_df.values, 1))
                        for corr_thereshold in [0.5, 0.6, 0.7, 0.8]:
                            big_corr_ratio = sum((corr_up_df >= corr_thereshold).sum()) / ((np.square(corr_up_df.shape[0]) - corr_up_df.shape[0]) / 2)
                            all_result_df.loc[date, f'{previous_day}days_corr>{corr_thereshold}_ratio'] = big_corr_ratio
                        # 分析PCA第一主成分贡献率
                        # modelPCA = PCA(n_components=3)  # 建立模型，设定保留主成分数 K=1
                        # try:
                        #     temp_ret_df.dropna(how='any', axis=1, inplace=True)  # 剔除掉为nan的数据，不然PCA.fit会报错
                        #     modelPCA.fit(temp_ret_df.values)  # 用数据集 X 训练 模型 modelPCA
                        #     all_result_df.loc[date, f'{previous_day}days_pca_1st_explained_ratio'] = modelPCA.explained_variance_ratio_[0]
                        #     all_result_df.loc[date, f'{previous_day}days_pca_2nd_explained_ratio'] = sum(modelPCA.explained_variance_ratio_[0:2])
                        #     all_result_df.loc[date, f'{previous_day}days_pca_3rd_explained_ratio'] = sum(modelPCA.explained_variance_ratio_[0:3])
                        # except:
                        #     pass

                # 币横截面走势分化度
                all_result_df.loc[date, f'whole_market_ret_divergence_{kick_small_amount}'] = np.std(temp_ret_series)

                # # 每天新上币数量；上市头一天（头一天的统计不到只能统计第二天的，因为没有开盘价），头一星期 ， 头1个月涨跌幅
                # temp_ico_date_series = ico_date_series[temp_amount_series.index]
                # all_result_df.loc[date, f'ico_coins_smalltrade_{kick_small_amount}'] = f'{temp_ico_date_series[temp_ico_date_series == date].index}'  # 这一天新上币的名称
                # all_result_df.loc[date, f'ico_num_smalltrade_{kick_small_amount}'] = len(temp_ico_date_series[temp_ico_date_series == date])  # 这一天新上币的个数
                # if ret_type == 'daily':
                #     yesterday = date_shifter(date, step='days', how_many=-1)
                #     yesterday_ico_coin = list(temp_ico_date_series[temp_ico_date_series == yesterday].index)  # todo 昨天新上的币，今天的收益率； 按道理应该统计的是上市那天的收益率，但是因为没有开盘价
                #     all_result_df.loc[date, f'ico_num_daily_ret_smalltrade_{kick_small_amount}'] = temp_ret_series[yesterday_ico_coin].mean()
                # if ret_type == 'weekly':
                #     week_ago = date_shifter(date, step='days', how_many=-7)
                #     week_ago_ico_coin = list(temp_ico_date_series[temp_ico_date_series == week_ago].index)  # 一个礼拜前新上的币，今天的周收益率
                #     all_result_df.loc[date, f'ico_num_weekly_ret_smalltrade_{kick_small_amount}'] = temp_ret_series[week_ago_ico_coin].mean()
                # if ret_type == 'monthly':
                #     month_ago = date_shifter(date, step='days', how_many=-30)
                #     month_ago_ico_coin = list(temp_ico_date_series[temp_ico_date_series == month_ago].index)  # 一个月前新上的币，今天的月收益率
                #     all_result_df.loc[date, f'ico_num_monthly_ret_smalltrade_{kick_small_amount}'] = temp_ret_series[month_ago_ico_coin].mean()

                # 统计所有符合条件的币，统计市值前100，前200的币的情况
                # for topcap_num in ['all', 'top100', 'top200']:
                for topcap_num in ['all']:
                    temp_cap_series_for_this_test = temp_cap_series.replace(0, np.nan)  # 剔除市值为0的
                    temp_cap_series_for_this_test.dropna(inplace=True)  # 剔除市值为空的
                    temp_cap_series_for_this_test.sort_values(ascending=False, inplace=True)  # 市值从大到小排列
                    if topcap_num == 'all':
                        all_coins_for_cal = temp_amount_series.index
                        temp_cap_series_new = temp_cap_series.copy()
                        temp_ret_series_new = temp_ret_series.copy()
                        temp_amount_series_new = temp_amount_series.copy()
                    elif topcap_num == 'top100':
                        all_coins_for_cal = temp_cap_series_for_this_test.index[:100]
                        temp_cap_series_new = temp_cap_series[all_coins_for_cal]
                        temp_ret_series_new = temp_ret_series[all_coins_for_cal]
                        temp_amount_series_new = temp_amount_series[all_coins_for_cal]
                    elif topcap_num == 'top200':
                        all_coins_for_cal = temp_cap_series_for_this_test.index[:200]
                        temp_cap_series_new = temp_cap_series[all_coins_for_cal]
                        temp_ret_series_new = temp_ret_series[all_coins_for_cal]
                        temp_amount_series_new = temp_amount_series[all_coins_for_cal]

                    # 阈值设定
                    osc_ret_gap = [float('-inf'), -1, -0.8, -0.6, -0.4, -0.3, -0.2, -0.15, -0.1, -0.05, 0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.8, 1, 1.5, 2, 3, 5, 10, float('inf')]
                    positive_ret_gap = [0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.8, 1, 1.5, 2, 3, 5, 10, float('inf')]
                    negative_ret_gap = [float('-inf'), -1, -0.8, -0.6, -0.5, -0.4, -0.3, -0.2, -0.15, -0.1, -0.05, 0]
                    osc_ret_gap_list = [[osc_ret_gap[i], osc_ret_gap[i + 1]] for i in range(len(osc_ret_gap) - 1)]
                    positive_ret_gap_list = [[positive_ret_gap[i], positive_ret_gap[i + 1]] for i in range(len(positive_ret_gap) - 1)]
                    negative_ret_gap_list = [[negative_ret_gap[i], negative_ret_gap[i + 1]] for i in range(len(negative_ret_gap) - 1)]

                    # 统计上传下穿均线的情况
                    for ma_period in [30, 60, 120, 155, 200, 300]:
                        if ma_period == 30:
                            temp_bias_df = all_bias30_df
                            temp_osc_df = all_osc30_df
                        elif ma_period == 60:
                            temp_bias_df = all_bias60_df
                            temp_osc_df = all_osc60_df
                        elif ma_period == 120:
                            temp_bias_df = all_bias120_df
                            temp_osc_df = all_osc120_df
                        elif ma_period == 155:
                            temp_bias_df = all_bias155_df
                            temp_osc_df = all_osc155_df
                        elif ma_period == 200:
                            temp_bias_df = all_bias200_df
                            temp_osc_df = all_osc200_df
                        elif ma_period == 300:
                            temp_bias_df = all_bias300_df
                            temp_osc_df = all_osc300_df
                        else:
                            raise NotImplementedError

                        bias_series = temp_bias_df.loc[date, all_coins_for_cal]
                        bias_series.dropna(inplace=True)
                        all_result_df.loc[date, f'{topcap_num}_bias{ma_period}_num_smalltrade_{kick_small_amount}'] = len(bias_series[bias_series > 0])
                        if len(bias_series) == 0:
                            all_result_df.loc[date, f'{topcap_num}_bias{ma_period}_num_ratio_smalltrade_{kick_small_amount}'] = np.nan
                        else:
                            all_result_df.loc[date, f'{topcap_num}_bias{ma_period}_num_ratio_smalltrade_{kick_small_amount}'] = len(bias_series[bias_series > 0]) / len(bias_series)

                        # 统计每天osc分布情况（根据不同的收益率区间划分）
                        osc_series = temp_osc_df.loc[date, all_coins_for_cal]
                        osc_series.dropna(inplace=True)
                        for gap in osc_ret_gap_list:
                            temp_osc_series = osc_series[(osc_series > gap[0]) & (osc_series <= gap[1])]
                            all_result_df.loc[date, f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}'] = len(temp_osc_series)
                        gap_columns_name = [f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}' for gap in osc_ret_gap_list]
                        all_result_df.loc[date, gap_columns_name] = all_result_df.loc[date, gap_columns_name] / all_result_df.loc[date, gap_columns_name].sum()

                    # 近1,3,4,6月绝对收益率分布情况
                    if ret_type == 'daily':
                        for highlow_period in [30, 90, 120, 180]:
                            if highlow_period == 30:
                                temp_monthly_ret_df = all_monthly_ret_df
                            elif highlow_period == 90:
                                temp_monthly_ret_df = all_3monthly_ret_df
                            elif highlow_period == 120:
                                temp_monthly_ret_df = all_4monthly_ret_df
                            elif highlow_period == 180:
                                temp_monthly_ret_df = all_6monthly_ret_df
                            else:
                                raise NotImplementedError

                            monthly_ret_series = temp_monthly_ret_df.loc[date, all_coins_for_cal]
                            monthly_ret_series.dropna(inplace=True)
                            for gap in osc_ret_gap_list:
                                temp_monthly_ret_series = monthly_ret_series[(monthly_ret_series > gap[0]) & (monthly_ret_series <= gap[1])]
                                all_result_df.loc[date, f'{topcap_num}_monthlyret{highlow_period}_{gap}_smalltrade_{kick_small_amount}'] = len(temp_monthly_ret_series)
                            gap_columns_name = [f'{topcap_num}_monthlyret{highlow_period}_{gap}_smalltrade_{kick_small_amount}' for gap in osc_ret_gap_list]
                            all_result_df.loc[date, gap_columns_name] = all_result_df.loc[date, gap_columns_name] / all_result_df.loc[date, gap_columns_name].sum()

                    # 自1, 3, 4, 6月高点回撤，低点上涨幅度分布
                    if ret_type == 'daily':
                        for highlow_period in [7, 30, 90, 120, 180]:
                            if highlow_period == 7:
                                temp_highret_df = all_high7ret_df
                                temp_lowret_df = all_low7ret_df
                            elif highlow_period == 30:
                                temp_highret_df = all_high30ret_df
                                temp_lowret_df = all_low30ret_df
                            elif highlow_period == 90:
                                temp_highret_df = all_high90ret_df
                                temp_lowret_df = all_low90ret_df
                            elif highlow_period == 120:
                                temp_highret_df = all_high120ret_df
                                temp_lowret_df = all_low120ret_df
                            elif highlow_period == 180:
                                temp_highret_df = all_high180ret_df
                                temp_lowret_df = all_low180ret_df
                            else:
                                raise NotImplementedError

                            # 统计高点回撤收益率分布
                            highret_series = temp_highret_df.loc[date, all_coins_for_cal]
                            highret_series.dropna(inplace=True)
                            for gap in negative_ret_gap_list:
                                temp_highret_series = highret_series[(highret_series > gap[0]) & (highret_series <= gap[1])]
                                all_result_df.loc[date, f'{topcap_num}_highret{highlow_period}_{gap}_smalltrade_{kick_small_amount}'] = len(temp_highret_series)
                            gap_columns_name = [f'{topcap_num}_highret{highlow_period}_{gap}_smalltrade_{kick_small_amount}' for gap in negative_ret_gap_list]
                            all_result_df.loc[date, gap_columns_name] = all_result_df.loc[date, gap_columns_name] / all_result_df.loc[date, gap_columns_name].sum()

                            # 统计低点回升收益率分布
                            lowret_series = temp_lowret_df.loc[date, all_coins_for_cal]
                            lowret_series.dropna(inplace=True)
                            for gap in positive_ret_gap_list:
                                temp_lowret_series = lowret_series[(lowret_series > gap[0]) & (lowret_series <= gap[1])]
                                all_result_df.loc[date, f'{topcap_num}_lowret{highlow_period}_{gap}_smalltrade_{kick_small_amount}'] = len(temp_lowret_series)
                            gap_columns_name = [f'{topcap_num}_lowret{highlow_period}_{gap}_smalltrade_{kick_small_amount}' for gap in positive_ret_gap_list]
                            all_result_df.loc[date, gap_columns_name] = all_result_df.loc[date, gap_columns_name] / all_result_df.loc[date, gap_columns_name].sum()

                    # 创1, 2, 3, 4, 6月新高新低占比分布
                    if ret_type == 'daily':
                        for highlow_period in [30, 60, 90, 120, 180]:
                            if highlow_period == 30:
                                temp_high_df = all_high30_df
                                temp_low_df = all_low30_df
                            elif highlow_period == 60:
                                temp_high_df = all_high60_df
                                temp_low_df = all_low60_df
                            elif highlow_period == 90:
                                temp_high_df = all_high90_df
                                temp_low_df = all_low90_df
                            elif highlow_period == 120:
                                temp_high_df = all_high120_df
                                temp_low_df = all_low120_df
                            elif highlow_period == 180:
                                temp_high_df = all_high180_df
                                temp_low_df = all_low180_df
                            else:
                                raise NotImplementedError

                            low_series = temp_low_df.loc[date, all_coins_for_cal]
                            low_series.dropna(inplace=True)
                            temp_low_series = low_series[low_series == 0]
                            all_result_df.loc[date, f'{topcap_num}_low{highlow_period}_num_ratio_smalltrade_{kick_small_amount}'] = len(temp_low_series) / len(low_series)

                            high_series = temp_high_df.loc[date, all_coins_for_cal]
                            high_series.dropna(inplace=True)
                            temp_high_series = high_series[high_series == 0]
                            all_result_df.loc[date, f'{topcap_num}_high{highlow_period}_num_ratio_smalltrade_{kick_small_amount}'] = len(temp_high_series) / len(high_series)

                    # 比BTC近1天，近一周，近1,3,6月收益率高的币占比，或低的币占比
                    if ret_type == 'daily':
                        for compare_period in ['daily', 'weekly', '1monthly', '3monthly', '6monthly']:
                            if compare_period == 'daily':
                                temp_compare_ret_series = all_daily_ret_df.loc[date, all_coins_for_cal]
                                threshold_list = [0, 0.001, 0.005, 0.01, 0.03, 0.05, 0.1]
                            elif compare_period == 'weekly':
                                temp_compare_ret_series = all_weekly_ret_df.loc[date, all_coins_for_cal]
                                threshold_list = [0, 0.01, 0.03, 0.05, 0.1, 0.15, 0.2]
                            elif compare_period == '1monthly':
                                temp_compare_ret_series = all_monthly_ret_df.loc[date, all_coins_for_cal]
                                threshold_list = [0, 0.05, 0.1, 0.15, 0.2, 0.3]
                            elif compare_period == '3monthly':
                                temp_compare_ret_series = all_3monthly_ret_df.loc[date, all_coins_for_cal]
                                threshold_list = [0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5]
                            elif compare_period == '6monthly':
                                temp_compare_ret_series = all_6monthly_ret_df.loc[date, all_coins_for_cal]
                                threshold_list = [0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5]
                            else:
                                raise NotImplementedError
                            temp_compare_ret_series.dropna(inplace=True)
                            if len(temp_compare_ret_series) == 0:
                                continue
                            else:
                                for threshold in threshold_list:
                                    bigger_ret_series = temp_compare_ret_series[temp_compare_ret_series > (temp_compare_ret_series['bitcoin'] + threshold)]
                                    all_result_df.loc[date, f'{topcap_num}_{compare_period}_ret_biggerthan_btc_{threshold}_ratio_smalltrade_{kick_small_amount}'] = len(bigger_ret_series) / len(temp_compare_ret_series)
                                for threshold in threshold_list:
                                    smaller_ret_series = temp_compare_ret_series[temp_compare_ret_series < (temp_compare_ret_series['bitcoin'] - threshold)]
                                    all_result_df.loc[date, f'{topcap_num}_{compare_period}_ret_smallerthan_btc_{-threshold}_ratio_smalltrade_{kick_small_amount}'] = len(smaller_ret_series) / len(temp_compare_ret_series)

                    # 比BTC近一周，近1,3,6月高点回撤多的币占比，或者低点上涨多的币占比
                    if ret_type == 'daily':
                        for compare_period in [7, 30, 90, 120, 180]:
                            if compare_period == 7:
                                temp_highret_compare_ret_series = all_high7ret_df.loc[date, all_coins_for_cal]
                                temp_lowret_compare_ret_series = all_low7ret_df.loc[date, all_coins_for_cal]
                                highret_threshold_list = [0.01, 0.03, 0.05, 0.1, 0.15, 0.2]
                                lowret_threshold_list = [0.01, 0.03, 0.05, 0.1, 0.15, 0.2]
                            elif compare_period == 30:
                                temp_highret_compare_ret_series = all_high30ret_df.loc[date, all_coins_for_cal]
                                temp_lowret_compare_ret_series = all_low30ret_df.loc[date, all_coins_for_cal]
                                highret_threshold_list = [0.03, 0.05, 0.1, 0.15, 0.2, 0.3]
                                lowret_threshold_list = [0.03, 0.05, 0.1, 0.15, 0.2, 0.3]
                            elif compare_period == 90:
                                temp_highret_compare_ret_series = all_high90ret_df.loc[date, all_coins_for_cal]
                                temp_lowret_compare_ret_series = all_low90ret_df.loc[date, all_coins_for_cal]
                                highret_threshold_list = [0.03, 0.05, 0.1, 0.15, 0.2, 0.3]
                                lowret_threshold_list = [0.05, 0.1, 0.15, 0.2, 0.3, 0.5]
                            elif compare_period == 120:
                                temp_highret_compare_ret_series = all_high120ret_df.loc[date, all_coins_for_cal]
                                temp_lowret_compare_ret_series = all_low120ret_df.loc[date, all_coins_for_cal]
                                highret_threshold_list = [0.03, 0.05, 0.1, 0.15, 0.2, 0.3]
                                lowret_threshold_list = [0.05, 0.1, 0.15, 0.2, 0.3, 0.5]
                            elif compare_period == 180:
                                temp_highret_compare_ret_series = all_high180ret_df.loc[date, all_coins_for_cal]
                                temp_lowret_compare_ret_series = all_low180ret_df.loc[date, all_coins_for_cal]
                                highret_threshold_list = [0.03, 0.05, 0.1, 0.15, 0.2, 0.3]
                                lowret_threshold_list = [0.05, 0.1, 0.15, 0.2, 0.3, 0.5]
                            else:
                                raise NotImplementedError

                            temp_highret_compare_ret_series.dropna(inplace=True)
                            if len(temp_highret_compare_ret_series) != 0:
                                for threshold in highret_threshold_list:
                                    smaller_ret_series = temp_highret_compare_ret_series[temp_highret_compare_ret_series < (temp_highret_compare_ret_series['bitcoin'] - threshold)]
                                    all_result_df.loc[date, f'{topcap_num}_{compare_period}_highret_smallerthan_btc_{-threshold}_ratio_smalltrade_{kick_small_amount}'] = len(smaller_ret_series) / len(temp_highret_compare_ret_series)
                            temp_lowret_compare_ret_series.dropna(inplace=True)
                            if len(temp_lowret_compare_ret_series) != 0:
                                for threshold in lowret_threshold_list:
                                    bigger_ret_series = temp_lowret_compare_ret_series[temp_lowret_compare_ret_series > (temp_lowret_compare_ret_series['bitcoin'] + threshold)]
                                    all_result_df.loc[date, f'{topcap_num}_{compare_period}_lowret_biggerthan_btc_{threshold}_ratio_smalltrade_{kick_small_amount}'] = len(bigger_ret_series) / len(temp_lowret_compare_ret_series)

                        # # 统计RSI指标分布
                        # for rsi_period in [30, 90, 180]:
                        #     for rsi_gap in [[20, 80], [30, 70], [40, 60]]:
                        #         if rsi_period == 30:
                        #             if rsi_gap == [20, 80]:
                        #                 temp_rsi_signal_df = all_rsi_signal_df_list[0]
                        #             elif rsi_gap == [30, 70]:
                        #                 temp_rsi_signal_df = all_rsi_signal_df_list[1]
                        #             elif rsi_gap == [40, 60]:
                        #                 temp_rsi_signal_df = all_rsi_signal_df_list[2]
                        #         elif rsi_period == 90:
                        #             if rsi_gap == [20, 80]:
                        #                 temp_rsi_signal_df = all_rsi_signal_df_list[3]
                        #             elif rsi_gap == [30, 70]:
                        #                 temp_rsi_signal_df = all_rsi_signal_df_list[4]
                        #             elif rsi_gap == [40, 60]:
                        #                 temp_rsi_signal_df = all_rsi_signal_df_list[5]
                        #         elif rsi_period == 180:
                        #             if rsi_gap == [20, 80]:
                        #                 temp_rsi_signal_df = all_rsi_signal_df_list[6]
                        #             elif rsi_gap == [30, 70]:
                        #                 temp_rsi_signal_df = all_rsi_signal_df_list[7]
                        #             elif rsi_gap == [40, 60]:
                        #                 temp_rsi_signal_df = all_rsi_signal_df_list[8]
                        #         else:
                        #             raise NotImplementedError
                        #
                        #         rsi_signal_series = temp_rsi_signal_df.loc[date, all_coins_for_cal]
                        #         rsi_signal_series.dropna(inplace=True)
                        #         all_result_df.loc[date, f'{topcap_num}_rsi{rsi_period}_{rsi_gap}_num_smalltrade_{kick_small_amount}'] = len(rsi_signal_series[rsi_signal_series == 1])  # 信号为1的个数
                        #         if len(rsi_signal_series) == 0:
                        #             all_result_df.loc[date, f'{topcap_num}_rsi{rsi_period}_{rsi_gap}_num_ratio_smalltrade_{kick_small_amount}'] = np.nan
                        #         else:
                        #             all_result_df.loc[date, f'{topcap_num}_rsi{rsi_period}_{rsi_gap}_num_ratio_smalltrade_{kick_small_amount}'] = len(rsi_signal_series[rsi_signal_series == 1]) / len(rsi_signal_series)

                    # 统计每天收益率分布情况（根据不同的收益率区间划分）
                    for gap in ret_gap_list:
                        temp_gap_ret_series = temp_ret_series_new[(temp_ret_series_new > gap[0]) & (temp_ret_series_new <= gap[1])]
                        all_result_df.loc[date, f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}'] = len(temp_gap_ret_series)
                    gap_columns_name = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}' for gap in ret_gap_list]
                    all_result_df.loc[date, f'{topcap_num}_all_coins_num_{kick_small_amount}'] = all_result_df.loc[date, gap_columns_name].sum()
                    all_result_df.loc[date, gap_columns_name] = all_result_df.loc[date, gap_columns_name] / all_result_df.loc[date, gap_columns_name].sum()

                    # 所有符合条件的币的名字(清洗之后按市值从大到小排序)
                    temp_cap_series_name = temp_cap_series_new.sort_values(ascending=False)  # 市值从大到小排列
                    all_result_df.at[date, f'{topcap_num}_all_cleaned_coins_name_{kick_small_amount}'] = list(temp_cap_series_name.index)

                    # # 统计n个最大收益率, n个最小收益率的简单平均，市值加权，交易额加权收益率
                    # group_num_list = [i for i in [3, 5, 10, 20, 30, 50] if i <= len(temp_ret_series_new)]
                    # for i in group_num_list:
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}_ret_name_smalltrade_{kick_small_amount}'] = f'{list(temp_ret_series_new[:i].index)}'  # n个最大收益率的币的名字
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}_ret_smalltrade_{kick_small_amount}'] = temp_ret_series_new[:i].mean()
                    #     if temp_cap_series_new[temp_ret_series_new[:i].index].sum() == 0:
                    #         all_result_df.loc[date, f'{topcap_num}_largest{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'] = np.nan
                    #     else:
                    #         all_result_df.loc[date, f'{topcap_num}_largest{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[:i] * temp_cap_series_new[temp_ret_series_new[:i].index]).sum() / temp_cap_series_new[
                    #             temp_ret_series_new[:i].index].sum()
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}_amount_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[:i] * temp_amount_series_new[temp_ret_series_new[:i].index]).sum() / temp_amount_series_new[
                    #         temp_ret_series_new[:i].index].sum()
                    #     all_result_df.loc[date, f'{topcap_num}_smallest{i}_ret_name_smalltrade_{kick_small_amount}'] = f'{list(temp_ret_series_new[-i:].index)}'  # n个最小收益率的币的名字
                    #     all_result_df.loc[date, f'{topcap_num}_smallest{i}_ret_smalltrade_{kick_small_amount}'] = temp_ret_series_new[-i:].mean()
                    #     if temp_cap_series_new[temp_ret_series_new[-i:].index].sum() == 0:
                    #         all_result_df.loc[date, f'{topcap_num}_smallest{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'] = np.nan
                    #     else:
                    #         all_result_df.loc[date, f'{topcap_num}_smallest{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[-i:] * temp_cap_series_new[temp_ret_series_new[-i:].index]).sum() / temp_cap_series_new[
                    #             temp_ret_series_new[-i:].index].sum()
                    #     all_result_df.loc[date, f'{topcap_num}_smallest{i}_amount_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[-i:] * temp_amount_series_new[temp_ret_series_new[-i:].index]).sum() / temp_amount_series_new[
                    #         temp_ret_series_new[-i:].index].sum()

                    # # 统计整体市值加权,交易额加权收益率
                    # all_result_df.loc[date, f'{topcap_num}_cap_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new * temp_cap_series_new).sum() / temp_cap_series_new.sum()
                    # all_result_df.loc[date, f'{topcap_num}_amount_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new * temp_amount_series_new).sum() / temp_amount_series_new.sum()

                    temp_cap_series_new = temp_cap_series_new.replace(0, np.nan)  # 剔除市值为0的
                    temp_cap_series_new.dropna(inplace=True)  # 剔除市值为空的
                    temp_cap_series_new.sort_values(ascending=False, inplace=True)  # 市值从大到小排列
                    temp_cap_series_new = temp_cap_series_new.astype(float)

                    # 每天市值前n的币有多少是之前也在前n的
                    for previous_day in [30, 90, 180]:
                        days_ago = date_shifter(date, step='days', how_many=-previous_day)
                        for top_num in [30, 50]:
                            if days_ago < earliest_day:
                                all_result_df.loc[date, f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago'] = np.nan
                            else:
                                # 当前市值前n的币的名称
                                top_coin_this_date = set(temp_cap_series_new[:top_num].index)
                                # n天前市值前n的币的名称
                                top_coin_days_ago = all_result_df.loc[days_ago, f'{topcap_num}_all_cleaned_coins_name_{kick_small_amount}'][:top_num]
                                # temp_cap_series_new_days_ago = all_marketcap_df.loc[days_ago, :]
                                # temp_cap_series_new_days_ago.dropna(inplace=True)  # 剔除市值为空的
                                # temp_cap_series_new_days_ago.sort_values(ascending=False, inplace=True)  # 市值从大到小排列
                                # top_coin_days_ago = set(temp_cap_series_new_days_ago[:top_num].index)

                                # 每天市值前n的币有多少是n天前在市值前n的
                                all_result_df.loc[date, f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago'] = len(top_coin_this_date.intersection(top_coin_days_ago))
                                all_result_df.loc[date, f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago_ratio'] = len(top_coin_this_date.intersection(top_coin_days_ago)) / len(top_coin_this_date)

                    # # 剔除市值前三, 前五的币后计算全市场市值加权和交易额加权收益率
                    # for kick_num in [3, 5]:
                    #     alter_coins_kick_largest = list(temp_cap_series_new.index[kick_num:])
                    #     temp_cap_series_new_altercoins = temp_cap_series_new[alter_coins_kick_largest]
                    #     temp_ret_series_new_altercoins = temp_ret_series_new[alter_coins_kick_largest]
                    #     temp_amount_series_new_altercoins = temp_amount_series_new[alter_coins_kick_largest]
                    #     all_result_df.loc[date, f'{topcap_num}_altercoins_kick_largest{kick_num}_cap_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new_altercoins * temp_cap_series_new_altercoins).sum() / temp_cap_series_new_altercoins.sum()
                    #     all_result_df.loc[date, f'{topcap_num}_altercoins_kick_largest{kick_num}_amount_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new_altercoins * temp_amount_series_new_altercoins).sum() / temp_amount_series_new_altercoins.sum()
                    #
                    # # 前n市值与后n市值的市值加权平均涨跌幅差异
                    # group_num_list = [i for i in [3, 5, 10, 20, 30, 50] if i <= len(temp_cap_series_new)]
                    # for i in group_num_list:
                    #     # 前n市值的币, 和市值最n小的币为
                    #     large_coins = list(temp_cap_series_new[:i].index)
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}cap_ret_name_smalltrade_{kick_small_amount}'] = f'{large_coins}'
                    #     small_coins = list(temp_cap_series_new[-i:].index)
                    #     all_result_df.loc[date, f'{topcap_num}_smallest{i}cap_ret_name_smalltrade_{kick_small_amount}'] = f'{small_coins}'
                    #     # 对应的简单平均, 市值加权，交易额加权
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}cap_ret_smalltrade_{kick_small_amount}'] = temp_ret_series_new[large_coins].mean()
                    #     all_result_df.loc[date, f'{topcap_num}_smallest{i}cap_ret_smalltrade_{kick_small_amount}'] = temp_ret_series_new[small_coins].mean()
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[large_coins] * temp_cap_series_new[large_coins]).sum() / temp_cap_series_new[large_coins].sum()
                    #     all_result_df.loc[date, f'{topcap_num}_smallest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[small_coins] * temp_cap_series_new[small_coins]).sum() / temp_cap_series_new[small_coins].sum()
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}cap_total_amount_smalltrade_{kick_small_amount}'] = temp_amount_series_new[large_coins].mean()
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[large_coins] * temp_amount_series_new[large_coins]).sum() / temp_amount_series_new[large_coins].sum()
                    #     all_result_df.loc[date, f'{topcap_num}_smallest{i}cap_total_amount_smalltrade_{kick_small_amount}'] = temp_amount_series_new[small_coins].mean()
                    #     all_result_df.loc[date, f'{topcap_num}_smallest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[small_coins] * temp_amount_series_new[small_coins]).sum() / temp_amount_series_new[small_coins].sum()
                    #     # 前n市值与后n市值的市值加权平均涨跌幅差异 和 简单平均收益率差异 和 交易额加权收益率差异
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}cap-smallest{i}cap_ret_smalltrade_{kick_small_amount}'] = all_result_df.loc[date, f'{topcap_num}_largest{i}cap_ret_smalltrade_{kick_small_amount}'] - all_result_df.loc[
                    #         date, f'{topcap_num}_smallest{i}cap_ret_smalltrade_{kick_small_amount}']
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}cap-smallest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}'] = all_result_df.loc[date, f'{topcap_num}_largest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}'] - \
                    #                                                                                                                             all_result_df.loc[date, f'{topcap_num}_smallest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}']
                    #     all_result_df.loc[date, f'{topcap_num}_largest{i}cap-smallest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}'] = all_result_df.loc[date, f'{topcap_num}_largest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}'] - \
                    #                                                                                                                                all_result_df.loc[date, f'{topcap_num}_smallest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}']
                    #
                    # # 分成BTC 和 其他币的收益率差别
                    # btc_coins = ['bitcoin']
                    # coins_except_btc = list(set(list(temp_cap_series_new.index)) ^ set(btc_coins))
                    # # 对应的简单平均, 市值加权，交易额加权
                    # all_result_df.loc[date, f'{topcap_num}_bitcoin_ret_smalltrade_{kick_small_amount}'] = temp_ret_series_new[btc_coins].mean()
                    # all_result_df.loc[date, f'{topcap_num}_altercoin_ret_smalltrade_{kick_small_amount}'] = temp_ret_series_new[coins_except_btc].mean()
                    # all_result_df.loc[date, f'{topcap_num}_bitcoin_cap_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[btc_coins] * temp_cap_series_new[btc_coins]).sum() / temp_cap_series_new[btc_coins].sum()
                    # all_result_df.loc[date, f'{topcap_num}_altercoin_cap_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[coins_except_btc] * temp_cap_series_new[coins_except_btc]).sum() / temp_cap_series_new[coins_except_btc].sum()
                    # all_result_df.loc[date, f'{topcap_num}_bitcoin_total_amount_smalltrade_{kick_small_amount}'] = temp_amount_series_new[btc_coins].sum()
                    # all_result_df.loc[date, f'{topcap_num}_bitcoin_amount_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[btc_coins] * temp_amount_series_new[btc_coins]).sum() / temp_amount_series_new[btc_coins].sum()
                    # all_result_df.loc[date, f'{topcap_num}_altercoin_total_amount_smalltrade_{kick_small_amount}'] = temp_amount_series_new[coins_except_btc].sum()
                    # all_result_df.loc[date, f'{topcap_num}_altercoin_amount_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[coins_except_btc] * temp_amount_series_new[coins_except_btc]).sum() / temp_amount_series_new[
                    #     coins_except_btc].sum()
                    # # 两组市值加权平均涨跌幅差异 和 简单平均收益率差异 和 交易额加权收益率差异
                    # all_result_df.loc[date, f'{topcap_num}_bitcoin-altercoin_ret_smalltrade_{kick_small_amount}'] = all_result_df.loc[date, f'{topcap_num}_bitcoin_ret_smalltrade_{kick_small_amount}'] - all_result_df.loc[
                    #     date, f'{topcap_num}_altercoin_ret_smalltrade_{kick_small_amount}']
                    # all_result_df.loc[date, f'{topcap_num}_bitcoin-altercoin_cap_weighted_ret_smalltrade_{kick_small_amount}'] = all_result_df.loc[date, f'{topcap_num}_bitcoin_cap_weighted_ret_smalltrade_{kick_small_amount}'] - all_result_df.loc[
                    #     date, f'{topcap_num}_altercoin_cap_weighted_ret_smalltrade_{kick_small_amount}']
                    # all_result_df.loc[date, f'{topcap_num}_bitcoin-altercoin_amount_weighted_ret_smalltrade_{kick_small_amount}'] = all_result_df.loc[date, f'{topcap_num}_bitcoin_amount_weighted_ret_smalltrade_{kick_small_amount}'] - all_result_df.loc[
                    #     date, f'{topcap_num}_altercoin_amount_weighted_ret_smalltrade_{kick_small_amount}']
                    #
                    # # 根据市值分为n组， 第一组和最后一组的市值加权平均涨跌幅差异
                    # if len(temp_cap_series_new) == 1:  # 当天有市值的币太少，不够分组
                    #     continue
                    # group_num_list = [i for i in [2, 3, 5, 10] if i <= len(temp_cap_series_new)]
                    # for i in group_num_list:
                    #     # quantile_cap_series = pd.qcut(temp_cap_series_new, i, labels=[group + 1 for group in range(i)]).astype(int)
                    #     quantile_cap_series = pd.qcut(temp_cap_series_new, i, labels=[group + 1 for group in range(i)], duplicates='drop').astype(int)
                    #     # quantile_cap_series = pd.qcut(temp_cap_series_new, i, labels=False, duplicates='drop').astype(int)  # label为整数，从0开始
                    #     group_num = quantile_cap_series.max()
                    #     for j in range(1, group_num + 1):
                    #         coins = list(quantile_cap_series[(quantile_cap_series == j)].index)
                    #         all_result_df.loc[date, f'{topcap_num}_group{i}{j}_ret_name_smalltrade_{kick_small_amount}'] = f'{coins}'
                    #         all_result_df.loc[date, f'{topcap_num}_group{i}{j}_ret_smalltrade_{kick_small_amount}'] = temp_ret_series_new[coins].mean()
                    #         all_result_df.loc[date, f'{topcap_num}_group{i}{j}_cap_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[coins] * temp_cap_series_new[coins]).sum() / temp_cap_series_new[coins].sum()
                    #         all_result_df.loc[date, f'{topcap_num}_group{i}{j}_total_amount_smalltrade_{kick_small_amount}'] = temp_amount_series_new[coins].mean()
                    #         all_result_df.loc[date, f'{topcap_num}_group{i}{j}_amount_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[coins] * temp_amount_series_new[coins]).sum() / temp_amount_series_new[coins].sum()
                    #     all_result_df.loc[date, f'{topcap_num}_group{i}_ret_diff_smalltrade_{kick_small_amount}'] = all_result_df.loc[date, f'{topcap_num}_group{i}{i}_ret_smalltrade_{kick_small_amount}'] - all_result_df.loc[
                    #         date, f'{topcap_num}_group{i}1_ret_smalltrade_{kick_small_amount}']
                    #     all_result_df.loc[date, f'{topcap_num}_group{i}_cap_weighted_ret_diff_smalltrade_{kick_small_amount}'] = all_result_df.loc[date, f'{topcap_num}_group{i}{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'] - all_result_df.loc[
                    #         date, f'{topcap_num}_group{i}1_cap_weighted_ret_smalltrade_{kick_small_amount}']
                    #     all_result_df.loc[date, f'{topcap_num}_group{i}_amount_weighted_ret_diff_smalltrade_{kick_small_amount}'] = all_result_df.loc[date, f'{topcap_num}_group{i}{i}_amount_weighted_ret_smalltrade_{kick_small_amount}'] - all_result_df.loc[
                    #         date, f'{topcap_num}_group{i}1_amount_weighted_ret_smalltrade_{kick_small_amount}']

                    # # 剔除某些币后，根据市值分为n组， 第一组和最后一组的市值加权平均涨跌幅差异
                    # # alter_coins_0 = list(set(list(temp_cap_series_new.index)).difference(set(['bitcoin', 'ethereum'])))  # 剔除比特币，以太坊
                    # alter_coins_0 = list(temp_cap_series_new.index[3:])  # 剔除市值前三的币
                    # alter_coins_1 = list(temp_cap_series_new.index[5:])  # 剔除市值前五的币
                    # # alter_coins_2 = list(temp_cap_series_new.index[:100])  # 统计市值前100的币，不剔除最大的那些市值
                    # # alter_coins_3 = list(temp_cap_series_new.index[:200])  # 统计市值前200的币，不剔除最大的那些市值
                    #
                    # for num, alter_coins in enumerate([alter_coins_0, alter_coins_1]):
                    #     if num == 0:
                    #         group_name = 'kick_cap3'
                    #     elif num == 1:
                    #         group_name = 'kick_cap5'
                    #     # elif num == 2:
                    #     #     group_name = 'cap100'
                    #     # elif num == 3:
                    #     #     group_name = 'cap200'
                    #     temp_cap_series_new_alter_coins = temp_cap_series_new[alter_coins]
                    #     temp_cap_series_new_alter_coins.sort_values(ascending=False, inplace=True)  # 市值从大到小排列
                    #
                    #     if len(temp_cap_series_new_alter_coins) in [0, 1]:  # 当天有市值的币太少，不够分组
                    #         continue
                    #     group_num_list = [i for i in [2, 3, 5, 10] if i <= len(temp_cap_series_new_alter_coins)]
                    #     for i in group_num_list:
                    #         # quantile_cap_series = pd.qcut(temp_cap_series_new_alter_coins, i, labels=[group + 1 for group in range(i)]).astype(int)
                    #         quantile_cap_series = pd.qcut(temp_cap_series_new_alter_coins, i, labels=[group + 1 for group in range(i)], duplicates='drop').astype(int)
                    #         # quantile_cap_series = pd.qcut(temp_cap_series_new_alter_coins, i, labels=False, duplicates='drop').astype(int)  # label为整数，从0开始
                    #         group_num = quantile_cap_series.max()
                    #         for j in range(1, group_num + 1):
                    #             coins = list(quantile_cap_series[(quantile_cap_series == j)].index)
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}{j}_ret_name_smalltrade_{kick_small_amount}'] = f'{topcap_num}_{coins}'
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}{j}_ret_smalltrade_{kick_small_amount}'] = temp_ret_series_new[coins].mean()
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}{j}_cap_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[coins] * temp_cap_series_new_alter_coins[coins]).sum() / temp_cap_series_new_alter_coins[coins].sum()
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}{j}_total_amount_smalltrade_{kick_small_amount}'] = temp_amount_series_new[coins].mean()
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}{j}_amount_weighted_ret_smalltrade_{kick_small_amount}'] = (temp_ret_series_new[coins] * temp_amount_series_new[coins]).sum() / temp_amount_series_new[coins].sum()
                    #         all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}_ret_diff_smalltrade_{kick_small_amount}'] = \
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}{i}_ret_smalltrade_{kick_small_amount}'] - \
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}1_ret_smalltrade_{kick_small_amount}']
                    #         all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}_cap_weighted_ret_diff_smalltrade_{kick_small_amount}'] = \
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'] - \
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}1_cap_weighted_ret_smalltrade_{kick_small_amount}']
                    #         all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}_amount_weighted_ret_diff_smalltrade_{kick_small_amount}'] = \
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}{i}_amount_weighted_ret_smalltrade_{kick_small_amount}'] - \
                    #             all_result_df.loc[date, f'{topcap_num}_{group_name}_group{i}1_amount_weighted_ret_smalltrade_{kick_small_amount}']

                    # 不同市值组别累计至今的累计收益率情况（累计1个月，3个月，6个月，12个月）
                    if ret_type == 'daily':
                        previous_365_day = date_shifter(date, step='days', how_many=-365)
                        previous_180_day = date_shifter(date, step='days', how_many=-180)
                        previous_90_day = date_shifter(date, step='days', how_many=-90)
                        previous_30_day = date_shifter(date, step='days', how_many=-30)

                        for previous_day in [30, 90, 180, 365]:
                            if previous_day == 30:
                                days_ago = previous_30_day
                            elif previous_day == 90:
                                days_ago = previous_90_day
                            elif previous_day == 180:
                                days_ago = previous_180_day
                            elif previous_day == 365:
                                days_ago = previous_365_day
                            else:
                                raise NotImplementedError
                            for top_num in [1, 3, 5, 10, 50, 100]:
                                if days_ago < earliest_day:
                                    all_result_df.loc[date, f'{topcap_num}_{top_num}_{previous_day}days_cumret'] = np.nan
                                else:
                                    # n天前市值前n的币的名称
                                    coins_days_ago = all_result_df.loc[days_ago, f'{topcap_num}_all_cleaned_coins_name_{kick_small_amount}']
                                    if len(coins_days_ago) < top_num:
                                        all_result_df.loc[date, f'{topcap_num}_{top_num}_{previous_day}days_cumret'] = np.nan
                                    else:
                                        if top_num == 1:
                                            top_coin_days_ago = coins_days_ago[0:1]
                                        elif top_num == 3:
                                            top_coin_days_ago = coins_days_ago[1:3]
                                        elif top_num == 5:
                                            top_coin_days_ago = coins_days_ago[3:5]
                                        elif top_num == 10:
                                            top_coin_days_ago = coins_days_ago[5:10]
                                        elif top_num == 50:
                                            top_coin_days_ago = coins_days_ago[10:50]
                                        elif top_num == 100:
                                            top_coin_days_ago = coins_days_ago[50:100]

                                        # # n天前市值前n的币的市值
                                        # top_coin_cap_days_ago = all_marketcap_df.loc[days_ago, top_coin_days_ago]
                                        # # n天前市值前n的币的市值
                                        # top_coin_amount_days_ago = all_amount_df.loc[days_ago, top_coin_days_ago]

                                        top_coin_ret_days_ago = all_ret_df.loc[days_ago:date, top_coin_days_ago].mean(axis=1)
                                        all_result_df.loc[date, f'{topcap_num}_{top_num}_{previous_day}days_cumret'] = (1 + top_coin_ret_days_ago).cumprod()[-1] - 1

                        if date > '2017-12-16':
                            for top_num in [1, 3, 5, 10, 50, 100]:
                                if date < '2021-04-13':
                                    coins_days_ago = all_result_df.loc['2017-12-16', f'{topcap_num}_all_cleaned_coins_name_{kick_small_amount}']
                                else:
                                    coins_days_ago = all_result_df.loc['2021-04-13', f'{topcap_num}_all_cleaned_coins_name_{kick_small_amount}']

                                if top_num == 1:
                                    top_coin_days_ago = coins_days_ago[0:1]
                                elif top_num == 3:
                                    top_coin_days_ago = coins_days_ago[1:3]
                                elif top_num == 5:
                                    top_coin_days_ago = coins_days_ago[3:5]
                                elif top_num == 10:
                                    top_coin_days_ago = coins_days_ago[5:10]
                                elif top_num == 50:
                                    top_coin_days_ago = coins_days_ago[10:50]
                                elif top_num == 100:
                                    top_coin_days_ago = coins_days_ago[50:100]

                                top_coin_drawdown_days_ago = all_maxdrawdown_df.loc[date, top_coin_days_ago].mean()
                                all_result_df.loc[date, f'{topcap_num}_{top_num}_maxdrawdown'] = top_coin_drawdown_days_ago

        all_df = pd.concat([all_result_df, price_log_price['log_prices']], axis=1)
        all_df.sort_index(inplace=True)
        all_df = all_df.loc[all_result_df.index[0]:, :]

        file_path = os.path.join(DATA_DIR, r'all_history_ohlcvm_coinmarketcap')
        file_name = os.path.join(file_path, f'All Coins {ret_type} Return Analysis')
        all_df.to_csv(f'{file_name}.csv')

        file_name = os.path.join(file_path, f'All Coins Name')  # 统计的所有币的范围
        all_coins_name_df.to_excel(f'{file_name}.xlsx')

        del all_df
        del all_result_df
        del all_coins_name_df


def all_ret_analysis():
    file_path = os.path.join(DATA_DIR, r'all_history_ohlcvm_coinmarketcap')

    # 获取BTC相关指标数据
    file_name_temp = os.path.join(DATA_DIR, f'BTC_history_ohlcvm')
    btc_ohlcv_data = pd.read_excel(f'{file_name_temp}.xlsx', index_col='end_date')
    btc_df = btc_ohlcv_data[['close']]
    btc_df['log_price'] = np.log10(btc_df['close'])
    for period in [7, 30, 60, 90, 120, 180]:
        btc_df[f'ma_{period}'] = btc_df['close'].rolling(period).mean()
        btc_df[f'btc_osc_{period}'] = (btc_df['close'] - btc_df[f'ma_{period}']) / btc_df[f'ma_{period}']
        # 区间收益率
        btc_df[f'btc_{period}days_ret'] = btc_df['close'].pct_change(period)
        # 高点回撤，低点上涨
        btc_df[f'btc_{period}days_highret'] = (btc_df['close'] - btc_df['close'].rolling(period).max()) / btc_df['close'].rolling(period).max()
        btc_df[f'btc_{period}days_lowret'] = (btc_df['close'] - btc_df['close'].rolling(period).min()) / btc_df['close'].rolling(period).min()
        # 创新高，创新低
        btc_df[f'btc_{period}days_newhigh'] = (btc_df['close'] - btc_df['close'].rolling(period).max())
        btc_df[f'btc_{period}days_newlow'] = (btc_df['close'] - btc_df['close'].rolling(period).min())

    def temp_data_analysis(temp_btc_df=pd.DataFrame(), column_name='', threshold=0, bigger_than=False, smaller_than=False, equal=False):
        temp_series = temp_btc_df[column_name]
        temp_data_analysis = temp_series.copy()
        if equal:
            temp_data_analysis[temp_series == threshold] = 5
            temp_data_analysis[temp_series != threshold] = 0
        if bigger_than:
            temp_data_analysis[temp_series >= threshold] = 5
            temp_data_analysis[temp_series < threshold] = 0
        if smaller_than:
            temp_data_analysis[temp_series <= threshold] = 5
            temp_data_analysis[temp_series > threshold] = 0

        return temp_data_analysis

    for ret_type in ['daily', 'weekly', 'monthly']:
        file_name = os.path.join(file_path, f'All Coins {ret_type} Return Analysis')
        ret_df = pd.read_csv(f'{file_name}.csv', index_col='end_date')
        ma_ret_df = pd.DataFrame(ret_df['log_prices'])

        temp_btc_df = btc_df.loc[ma_ret_df.index, :]  # BTC的相关指标

        if ret_type == 'daily':
            ret_gap = [float('-inf'), -1, -0.9, -0.8, -0.7, -0.6, -0.5, -0.4, -0.3, -0.2, -0.15, -0.1, -0.05, 0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.5, 2, 3, 5, 10, float('inf')]
        elif ret_type == 'weekly':
            ret_gap = [float('-inf'), -1, -0.9, -0.8, -0.7, -0.6, -0.5, -0.4, -0.3, -0.2, -0.15, -0.1, -0.05, 0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.5, 2, 3, 5, 10, float('inf')]
        elif ret_type == 'monthly':
            ret_gap = [float('-inf'), -1, -0.9, -0.8, -0.7, -0.6, -0.5, -0.4, -0.3, -0.2, -0.15, -0.1, -0.05, 0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.5, 2, 3, 5, 10, float('inf')]
        else:
            raise NotImplementedError
        ret_gap_list = [[ret_gap[i], ret_gap[i + 1]] for i in range(len(ret_gap) - 1)]

        osc_ret_gap = [float('-inf'), -1, -0.8, -0.6, -0.4, -0.3, -0.2, -0.15, -0.1, -0.05, 0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.8, 1, 1.5, 2, 3, 5, 10, float('inf')]
        positive_ret_gap = [0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.8, 1, 1.5, 2, 3, 5, 10, float('inf')]
        negative_ret_gap = [float('-inf'), -1, -0.8, -0.6, -0.5, -0.4, -0.3, -0.2, -0.15, -0.1, -0.05, 0]
        osc_ret_gap_list = [[osc_ret_gap[i], osc_ret_gap[i + 1]] for i in range(len(osc_ret_gap) - 1)]
        positive_ret_gap_list = [[positive_ret_gap[i], positive_ret_gap[i + 1]] for i in range(len(positive_ret_gap) - 1)]
        negative_ret_gap_list = [[negative_ret_gap[i], negative_ret_gap[i + 1]] for i in range(len(negative_ret_gap) - 1)]

        for ma in [7]:  # 以下分析的是不同指标的绝对值与行情指标的关系， 指标数据平滑的窗口可以选择
            # for ma in [7, 14, 30]:  # 数据平滑的窗口可以选择
            ma_ret_df[f'filtered_all_coin_num'] = ret_df[f'all_all_coins_num_True']
            # for kick_small_amount in [True, False]:
            for kick_small_amount in [True]:
                for topcap_num in ['all']:  # 统计币的范围，是全部，还是前100，前200
                    # for topcap_num in ['all', 'top100', 'top200']:
                    # for ma_period in [30, 60, 120, 155, 200, 300]:  # 上穿下穿均线情况统计，上下穿多长的均线
                    for ma_period in [120]:
                        for gap in osc_ret_gap_list:
                            if f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}' not in ret_df.columns:
                                continue
                            else:
                                ma_ret_df[f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                        columns_name_list_0 = [f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] >= 0]
                        ma_ret_df[f'{topcap_num}_osc{ma_period}_>0_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_0].sum(axis=1)
                        # BTC上穿下穿时间线
                        ma_ret_df[f'btc_osc{ma_period}_>0'] = temp_data_analysis(temp_btc_df, column_name=f'btc_osc_{ma_period}', threshold=0, bigger_than=True, smaller_than=False, equal=False)
                        columns_name_list_2 = [f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] >= 0.2]
                        ma_ret_df[f'{topcap_num}_osc{ma_period}_>0.2_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_2].sum(axis=1)
                        columns_name_list_4 = [f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] >= 0.4]
                        ma_ret_df[f'{topcap_num}_osc{ma_period}_>0.4_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_4].sum(axis=1)
                        columns_name_list_minus2 = [f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] <= -0.2]
                        ma_ret_df[f'{topcap_num}_osc{ma_period}_<minus0.2_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus2].sum(axis=1)
                        columns_name_list_minus4 = [f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] <= -0.4]
                        ma_ret_df[f'{topcap_num}_osc{ma_period}_<minus0.4_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus4].sum(axis=1)
                        all_columns = [f'{topcap_num}_osc{ma_period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list]
                        ma_ret_df.drop(columns=all_columns, inplace=True)

                    # 每日日收益率，周收益率，月收益分布数据平滑
                    for gap in ret_gap_list:
                        ma_ret_df[f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    columns_name_list_0 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] > 0]
                    ma_ret_df[f'{topcap_num}_>0_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_0].sum(axis=1)
                    columns_name_list_1 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] > 0.1]
                    ma_ret_df[f'{topcap_num}_>0.1_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_1].sum(axis=1)
                    columns_name_list_2 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] > 0.2]
                    ma_ret_df[f'{topcap_num}_>0.2_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_2].sum(axis=1)
                    columns_name_list_4 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] > 0.4]
                    ma_ret_df[f'{topcap_num}_>0.4_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_4].sum(axis=1)
                    columns_name_list_5 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] > 0.5]
                    ma_ret_df[f'{topcap_num}_>0.5_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_5].sum(axis=1)
                    columns_name_list_6 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] > 1]
                    ma_ret_df[f'{topcap_num}_>1_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_6].sum(axis=1)
                    columns_name_list_7 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] > 3]
                    ma_ret_df[f'{topcap_num}_>3_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_7].sum(axis=1)
                    columns_name_list_8 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] > 5]
                    ma_ret_df[f'{topcap_num}_>5_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_8].sum(axis=1)
                    columns_name_list_minus1 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] <= -0.1]
                    ma_ret_df[f'{topcap_num}_<minus0.1_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus1].sum(axis=1)
                    columns_name_list_minus2 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] <= -0.2]
                    ma_ret_df[f'{topcap_num}_<minus0.2_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus2].sum(axis=1)
                    columns_name_list_minus4 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] <= -0.4]
                    ma_ret_df[f'{topcap_num}_<minus0.4_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus4].sum(axis=1)
                    columns_name_list_minus5 = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list if gap[1] <= -0.5]
                    ma_ret_df[f'{topcap_num}_<minus0.5_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus5].sum(axis=1)
                    all_columns = [f'{topcap_num}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in ret_gap_list]
                    ma_ret_df.drop(columns=all_columns, inplace=True)

                    # 统计每天创新高创新低分布情况
                    if ret_type == 'daily':
                        # for highlow_period in [30, 60, 90, 120, 180]:
                        for highlow_period in [90]:
                            ma_ret_df[f'all_high{highlow_period}_num_ratio_smalltrade_True'] = ret_df[f'all_high{highlow_period}_num_ratio_smalltrade_True']
                            # ma_ret_df[f'btc_newhigh{highlow_period}'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{highlow_period}days_newhigh', threshold=0, bigger_than=False, smaller_than=False, equal=True)
                            ma_ret_df[f'all_low{highlow_period}_num_ratio_smalltrade_True'] = ret_df[f'all_low{highlow_period}_num_ratio_smalltrade_True']
                            # ma_ret_df[f'btc_newlow{highlow_period}'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{highlow_period}days_newlow', threshold=0, bigger_than=False, smaller_than=False, equal=True)

                    # 统计全市场横截面波动率
                    ma_ret_df[f'whole_market_ret_divergence_True'] = ret_df[f'whole_market_ret_divergence_True']

                    # 统计每天1,3, 4, 6月绝对收益率分布情况（均线处理）
                    if ret_type == 'daily':
                        for period in [30, 90, 120, 180]:
                            for gap in osc_ret_gap_list:
                                ma_ret_df[f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                            columns_name_list_1 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] > 0.1]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}>0.1_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_1].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret>0.1'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=0.1, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_2 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] > 0.2]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}>0.2_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_2].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret>0.2'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=0.2, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_4 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] > 0.4]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}>0.4_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_4].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret>0.4'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=0.4, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_5 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] > 0.5]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}>0.5_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_5].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret>0.5'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=0.5, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_6 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] > 1]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}>1_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_6].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret>1'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=1, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_7 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] > 3]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}>3_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_7].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret>3'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=3, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_8 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] > 5]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}>5_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_8].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret>5'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=5, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_minus1 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] <= -0.1]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}<minus0.1_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus1].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret<minus0.1'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=-0.1, bigger_than=False, smaller_than=True, equal=False)
                            columns_name_list_minus2 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] <= -0.2]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}<minus0.2_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus2].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret<minus0.2'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=-0.2, bigger_than=False, smaller_than=True, equal=False)
                            columns_name_list_minus4 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] <= -0.4]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}<minus0.4_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus4].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret<minus0.4'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=-0.4, bigger_than=False, smaller_than=True, equal=False)
                            columns_name_list_minus5 = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list if gap[1] <= -0.5]
                            ma_ret_df[f'{topcap_num}_monthlyret{period}<minus0.5_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus5].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_ret<minus0.5'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_ret', threshold=-0.5, bigger_than=False, smaller_than=True, equal=False)
                            all_columns = [f'{topcap_num}_monthlyret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in osc_ret_gap_list]
                            ma_ret_df.drop(columns=all_columns, inplace=True)


                    # 统计与BTC相比，近1天，近一周，近1,3,6月相对收益率分布情况（均线处理）
                    if ret_type == 'daily':
                        for compare_period in ['daily', 'weekly', '1monthly', '3monthly', '6monthly']:
                            if compare_period == 'daily':
                                threshold_list = [0, 0.001, 0.005, 0.01, 0.03, 0.05, 0.1]
                            elif compare_period == 'weekly':
                                threshold_list = [0, 0.01, 0.03, 0.05, 0.1, 0.15, 0.2]
                            elif compare_period == '1monthly':
                                threshold_list = [0, 0.05, 0.1, 0.15, 0.2, 0.3]
                            elif compare_period == '3monthly':
                                threshold_list = [0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5]
                            elif compare_period == '6monthly':
                                threshold_list = [0, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5]
                            else:
                                raise NotImplementedError
                            for threshold in threshold_list:
                                ma_ret_df[f'all_{compare_period}_ret_biggerthan_btc_{threshold}_ratio_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'all_{compare_period}_ret_biggerthan_btc_{threshold}_ratio_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                            for threshold in threshold_list:
                                ma_ret_df[f'all_{compare_period}_ret_smallerthan_btc_{-threshold}_ratio_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'all_{compare_period}_ret_smallerthan_btc_{-threshold}_ratio_smalltrade_{kick_small_amount}'].rolling(ma).mean()


                    # 统计每天近1周，近1,3,4,6月高点回撤，低点上涨收益率分布情况（均线处理）
                    if ret_type == 'daily':
                        for period in [7, 30, 90, 120, 180]:
                        # for period in [90]:
                            # 统计高点回撤收益率分布
                            for gap in negative_ret_gap_list:
                                ma_ret_df[f'{topcap_num}_highret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_highret{period}_{gap}_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                            columns_name_list_minus1 = [f'{topcap_num}_highret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in negative_ret_gap_list if gap[1] <= -0.1]
                            ma_ret_df[f'{topcap_num}_highret{period}<minus0.1_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus1].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_highret<minus0.1'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_highret', threshold=-0.1, bigger_than=False, smaller_than=True, equal=False)
                            columns_name_list_minus2 = [f'{topcap_num}_highret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in negative_ret_gap_list if gap[1] <= -0.2]
                            ma_ret_df[f'{topcap_num}_highret{period}<minus0.2_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus2].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_highret<minus0.2'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_highret', threshold=-0.2, bigger_than=False, smaller_than=True, equal=False)
                            columns_name_list_minus4 = [f'{topcap_num}_highret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in negative_ret_gap_list if gap[1] <= -0.4]
                            ma_ret_df[f'{topcap_num}_highret{period}<minus0.4_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus4].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_highret<minus0.4'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_highret', threshold=-0.4, bigger_than=False, smaller_than=True, equal=False)
                            columns_name_list_minus5 = [f'{topcap_num}_highret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in negative_ret_gap_list if gap[1] <= -0.5]
                            ma_ret_df[f'{topcap_num}_highret{period}<minus0.5_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_minus5].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_highret<minus0.5'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_highret', threshold=-0.5, bigger_than=False, smaller_than=True, equal=False)
                            all_columns = [f'{topcap_num}_highret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in negative_ret_gap_list]
                            ma_ret_df.drop(columns=all_columns, inplace=True)
                            # 统计低点回升收益率分布
                            for gap in positive_ret_gap_list:
                                ma_ret_df[f'{topcap_num}_lowret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_lowret{period}_{gap}_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                            columns_name_list_1 = [f'{topcap_num}_lowret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in positive_ret_gap_list if gap[1] > 0.1]
                            ma_ret_df[f'{topcap_num}_lowret{period}>0.1_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_1].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_lowret>0.1'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_lowret', threshold=0.1, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_2 = [f'{topcap_num}_lowret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in positive_ret_gap_list if gap[1] > 0.2]
                            ma_ret_df[f'{topcap_num}_lowret{period}>0.2_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_2].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_lowret>0.2'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_lowret', threshold=0.2, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_4 = [f'{topcap_num}_lowret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in positive_ret_gap_list if gap[1] > 0.4]
                            ma_ret_df[f'{topcap_num}_lowret{period}>0.4_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_4].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_lowret>0.4'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_lowret', threshold=0.4, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_5 = [f'{topcap_num}_lowret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in positive_ret_gap_list if gap[1] > 0.5]
                            ma_ret_df[f'{topcap_num}_lowret{period}>0.5_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_5].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_lowret>0.5'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_lowret', threshold=0.5, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_6 = [f'{topcap_num}_lowret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in positive_ret_gap_list if gap[1] > 1]
                            ma_ret_df[f'{topcap_num}_lowret{period}>1_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_6].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_lowret>1'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_lowret', threshold=1, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_7 = [f'{topcap_num}_lowret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in positive_ret_gap_list if gap[1] > 3]
                            ma_ret_df[f'{topcap_num}_lowret{period}>3_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_7].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_lowret>3'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_lowret', threshold=3, bigger_than=True, smaller_than=False, equal=False)
                            columns_name_list_8 = [f'{topcap_num}_lowret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in positive_ret_gap_list if gap[1] > 5]
                            ma_ret_df[f'{topcap_num}_lowret{period}>5_smalltrade_{kick_small_amount}_ma{ma}'] = ma_ret_df[columns_name_list_8].sum(axis=1)
                            # ma_ret_df[f'btc_{period}days_lowret>5'] = temp_data_analysis(temp_btc_df, column_name=f'btc_{period}days_lowret', threshold=5, bigger_than=True, smaller_than=False, equal=False)
                            all_columns = [f'{topcap_num}_lowret{period}_{gap}_smalltrade_{kick_small_amount}_ma{ma}' for gap in positive_ret_gap_list]
                            ma_ret_df.drop(columns=all_columns, inplace=True)

                    # 统计与BTC相比，每天近1周，近1,3,4,6月高点回撤，低点上涨相对收益率分布情况（均线处理）
                    if ret_type == 'daily':
                        for compare_period in [7, 30, 90, 120, 180]:
                            if compare_period == 7:
                                highret_threshold_list = [0.01, 0.03, 0.05, 0.1, 0.15, 0.2]
                                lowret_threshold_list = [0.01, 0.03, 0.05, 0.1, 0.15, 0.2]
                            elif compare_period == 30:
                                highret_threshold_list = [0.03, 0.05, 0.1, 0.15, 0.2, 0.3]
                                lowret_threshold_list = [0.03, 0.05, 0.1, 0.15, 0.2, 0.3]
                            elif compare_period == 90:
                                highret_threshold_list = [0.03, 0.05, 0.1, 0.15, 0.2, 0.3]
                                lowret_threshold_list = [0.05, 0.1, 0.15, 0.2, 0.3, 0.5]
                            elif compare_period == 120:
                                highret_threshold_list = [0.03, 0.05, 0.1, 0.15, 0.2, 0.3]
                                lowret_threshold_list = [0.05, 0.1, 0.15, 0.2, 0.3, 0.5]
                            elif compare_period == 180:
                                highret_threshold_list = [0.03, 0.05, 0.1, 0.15, 0.2, 0.3]
                                lowret_threshold_list = [0.05, 0.1, 0.15, 0.2, 0.3, 0.5]
                            else:
                                raise NotImplementedError

                            for threshold in highret_threshold_list:
                                ma_ret_df[f'all_{compare_period}_highret_smallerthan_btc_{-threshold}_ratio_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'all_{compare_period}_highret_smallerthan_btc_{-threshold}_ratio_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                            for threshold in lowret_threshold_list:
                                ma_ret_df[f'all_{compare_period}_lowret_biggerthan_btc_{threshold}_ratio_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'all_{compare_period}_lowret_biggerthan_btc_{threshold}_ratio_smalltrade_{kick_small_amount}'].rolling(ma).mean()

                    # # ICO数据平滑
                    # ma_ret_df[f'ico_num_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'ico_num_smalltrade_{kick_small_amount}'].rolling(ma).mean()

                    # # 统计整体市值加权,交易额加权收益率数据平滑
                    # ma_ret_df[f'{topcap_num}_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # ma_ret_df[f'{topcap_num}_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()

                    # 50 in 50 数据（均线处理）
                    # for previous_day in [30, 90, 180]:
                    #     for top_num in [30, 50]:
                    for previous_day in [180]:
                        for top_num in [50]:
                            # 均线前数据
                            ma_ret_df[f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago'] = ret_df[f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago']
                            ma_ret_df[f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago_ratio'] = ret_df[f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago_ratio']
                            ma_ret_df[f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago_ma{ma}'] = ret_df[f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago'].rolling(ma).mean()
                            ma_ret_df[f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago_ratio_ma{ma}'] = ret_df[f'{topcap_num}_{top_num}_in_{top_num}_{previous_day}daysago_ratio'].rolling(ma).mean()

                    # 不同市值组别不同时间区间的累计收益率差异, 不同市值组别的最大回撤数据（均线处理）
                    if ret_type == 'daily':
                        # for previous_day in [30, 90, 180, 365]:
                        for previous_day in [90]:
                            for top_num in [1, 3, 5, 10, 50, 100]:
                                ma_ret_df[f'{topcap_num}_{top_num}_{previous_day}days_cumret'] = ret_df[f'{topcap_num}_{top_num}_{previous_day}days_cumret']  # 均线前数据
                                ma_ret_df[f'{topcap_num}_{top_num}_{previous_day}days_cumret'][ma_ret_df[f'{topcap_num}_{top_num}_{previous_day}days_cumret'] > 0] = 0  # 为了画图好看，把收益率大于0的剔除掉
                                # ma_ret_df[f'{topcap_num}_{top_num}_{previous_day}days_cumret_ma{ma}'] = ret_df[f'{topcap_num}_{top_num}_{previous_day}days_cumret'].rolling(ma).mean()

                        for top_num in [1, 3, 5, 10, 50, 100]:
                            ma_ret_df[f'{topcap_num}_{top_num}_maxdrawdown'] = ret_df[f'{topcap_num}_{top_num}_maxdrawdown']  # 均线前数据
                            ma_ret_df[f'{topcap_num}_{top_num}_maxdrawdown_ma{ma}'] = ret_df[f'{topcap_num}_{top_num}_maxdrawdown'].rolling(ma).mean()

                    # for i in [3, 5]:
                    #     ma_ret_df[f'{topcap_num}_altercoins_kick_largest{i}_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_altercoins_kick_largest{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_altercoins_kick_largest{i}_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_altercoins_kick_largest{i}_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #
                    # # 统计n个最大收益率, n个最小收益率的简单平均，市值加权，交易额加权收益率
                    # for i in [3, 5, 10, 20, 30, 50]:
                    #     ma_ret_df[f'{topcap_num}_largest{i}_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_largest{i}_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_largest{i}_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_largest{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_largest{i}_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_largest{i}_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_smallest{i}_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_smallest{i}_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_smallest{i}_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_smallest{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_smallest{i}_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_smallest{i}_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #
                    # for i in [3, 5, 10, 20, 30, 50]:
                    #     # 前n市值的币, 和市值最n小的币对应的简单平均, 市值加权，交易额加权
                    #     ma_ret_df[f'{topcap_num}_largest{i}cap_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_largest{i}cap_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_largest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_largest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_largest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_largest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_largest{i}cap_total_amount_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_largest{i}cap_total_amount_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_smallest{i}cap_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_smallest{i}cap_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_smallest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_smallest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_smallest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_smallest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_smallest{i}cap_total_amount_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_smallest{i}cap_total_amount_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     # 前n市值与后n市值的市值加权平均涨跌幅差异 和 简单平均收益率差异 和 交易额加权收益率差异
                    #     ma_ret_df[f'{topcap_num}_largest{i}cap-smallest{i}cap_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_largest{i}cap-smallest{i}cap_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_largest{i}cap-smallest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_largest{i}cap-smallest{i}cap_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_largest{i}cap-smallest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_largest{i}cap-smallest{i}cap_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()

                    # 前n市值的币, 和市值最n小的币对应的简单平均, 市值加权，交易额加权
                    # ma_ret_df[f'{topcap_num}_bitcoin_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_bitcoin_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # ma_ret_df[f'{topcap_num}_bitcoin_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_bitcoin_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # ma_ret_df[f'{topcap_num}_bitcoin_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_bitcoin_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # ma_ret_df[f'{topcap_num}_bitcoin_total_amount_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_bitcoin_total_amount_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # ma_ret_df[f'{topcap_num}_altercoin_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_altercoin_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # ma_ret_df[f'{topcap_num}_altercoin_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_altercoin_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # ma_ret_df[f'{topcap_num}_altercoin_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_altercoin_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # ma_ret_df[f'{topcap_num}_altercoin_total_amount_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_altercoin_total_amount_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # # 前n市值与后n市值的市值加权平均涨跌幅差异 和 简单平均收益率差异 和 交易额加权收益率差异
                    # ma_ret_df[f'{topcap_num}_bitcoin-altercoin_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_bitcoin-altercoin_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # ma_ret_df[f'{topcap_num}_bitcoin-altercoin_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_bitcoin-altercoin_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    # ma_ret_df[f'{topcap_num}_bitcoin-altercoin_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_bitcoin-altercoin_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()

                    # for i in [2, 3, 5, 10]:
                    #     ma_ret_df[f'{topcap_num}_group{i}{i}_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}{i}_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_group{i}{i}_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_group{i}{i}_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}{i}_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_group{i}{i}_total_amount_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}{i}_total_amount_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_group{i}1_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}1_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_group{i}1_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}1_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_group{i}1_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}1_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_group{i}1_total_amount_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}1_total_amount_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_group{i}_ret_diff_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}_ret_diff_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_group{i}_cap_weighted_ret_diff_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}_cap_weighted_ret_diff_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #     ma_ret_df[f'{topcap_num}_group{i}_amount_weighted_ret_diff_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_group{i}_amount_weighted_ret_diff_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #
                    #     for group_name in ['kick_cap3', 'kick_cap5']:
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}{i}_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}{i}_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}{i}_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}{i}_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}{i}_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}{i}_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}{i}_total_amount_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}{i}_total_amount_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}1_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}1_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}1_cap_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}1_cap_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}1_amount_weighted_ret_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}1_amount_weighted_ret_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}1_total_amount_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}1_total_amount_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}_ret_diff_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}_ret_diff_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}_cap_weighted_ret_diff_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}_cap_weighted_ret_diff_smalltrade_{kick_small_amount}'].rolling(ma).mean()
                    #         ma_ret_df[f'{topcap_num}_{group_name}_group{i}_amount_weighted_ret_diff_smalltrade_{kick_small_amount}_ma{ma}'] = ret_df[f'{topcap_num}_{group_name}_group{i}_amount_weighted_ret_diff_smalltrade_{kick_small_amount}'].rolling(ma).mean()

        # for kick_small_amount in [True]:
        #     for topcap_num in ['all']:  # 统计币的范围，是全部，还是前100，前200
        #     # for topcap_num in ['all', 'top100', 'top200']:
        #         for ma_period in [30, 60, 120, 155, 200, 300]:
        #             for ma1 in range(10, 130, 20):  # 以下分析的是不同指标的双均线交叉跟行情买卖点关系
        #                 ma_ret_df[f'{topcap_num}_bias{ma_period}_num_ratio_smalltrade_{kick_small_amount}_ma{ma1}'] = ret_df[f'{topcap_num}_bias{ma_period}_num_ratio_smalltrade_{kick_small_amount}'].rolling(ma1).mean()
        #                 for ma2 in range(5, ma1, 10):
        #                     ma_ret_df[f'{topcap_num}_bias{ma_period}_num_ratio_smalltrade_{kick_small_amount}_ma{ma1}_ma{ma2}'] = ma_ret_df[f'{topcap_num}_bias{ma_period}_num_ratio_smalltrade_{kick_small_amount}_ma{ma1}'].rolling(ma2).mean()

        file_name = os.path.join(file_path, f'All Coins {ret_type} Return Analysis')
        ma_ret_df = ma_ret_df.loc['2017-01-01':, ]
        ma_ret_df.to_excel(f'{file_name}.xlsx')


if __name__ == '__main__':
    file_path = os.path.join(DATA_DIR, f'trading_data')
    os.makedirs(file_path, exist_ok=True)

    # 获取所有币的名单
    logger.info("开始get_all_coinlist_via_http, tag=all")
    task_to_db(os.path.basename(__file__), 'get_all_coinlist_via_http', 0, 'all')
    try:
        data = get_all_coinlist_via_http(status='inactive,active,untracked', method='http', tag='all')
    except Exception as e:
        msg = traceback.format_exc()
        logger.info(msg)
        send_error_to_email(script_name=os.path.basename(__file__), func_name="get_all_coinlist_via_http", message=msg)
        raise e
    task_to_db(os.path.basename(__file__), 'get_all_coinlist_via_http', 1, 'all')
    file_path = os.path.join(DATA_DIR, 'all_history_ohlcvm_coinmarketcap')
    file_name = os.path.join(file_path, f'all_coin_list_coinmarketcap')
    data[0].to_excel(f'{file_name}.xlsx')
    file_name = os.path.join(file_path, f'all_coin_tag_list_coinmarketcap')
    data[1].to_excel(f'{file_name}.xlsx')
    logger.info("成功get_all_coinlist_via_http, tag=all")

    # 获取所有稳定币的名单
    logger.info("开始get_all_coinlist_via_http, tag=stablecoin")
    task_to_db(os.path.basename(__file__), 'get_all_coinlist_via_http', 0, 'stablecoin')
    try:
        data = get_all_coinlist_via_http(status='inactive,active,untracked', method='http', tag='stablecoin')
    except Exception as e:
        msg = traceback.format_exc()
        logger.info(msg)
        send_error_to_email(script_name=os.path.basename(__file__), func_name="get_all_coinlist_via_http", message=msg)
        raise e
    task_to_db(os.path.basename(__file__), 'get_all_coinlist_via_http', 1, 'stablecoin')
    file_path = os.path.join(DATA_DIR, 'all_history_ohlcvm_coinmarketcap')
    file_name = os.path.join(file_path, f'all_stablecoin_list_coinmarketcap')
    data[0].to_excel(f'{file_name}.xlsx')
    logger.info("成功get_all_coinlist_via_http, tag=stablecoin")

    # 获取all_ohlcvm数据
    logger.info("开始get_all_ohlcvm_data")
    task_to_db(os.path.basename(__file__), 'get_all_ohlcvm_data')
    try:
        get_all_ohlcvm_data(data_source='coinmarketcap', asset='all', end_date=None, if_print=False, initialize=False, gap_days=0)
    except Exception as e:
        msg = traceback.format_exc()
        logger.info(msg)
        send_error_to_email(script_name=os.path.basename(__file__), func_name="get_all_ohlcvm_data", message=msg)
        raise e
    task_to_db(os.path.basename(__file__), 'get_all_ohlcvm_data', 1)
    logger.info("成功get_all_ohlcvm_data")

    # 计算市值占比数据，统计涨跌幅数据
    logger.info("开始get_trading_data, asset=BTC")
    task_to_db(os.path.basename(__file__), 'get_trading_data')
    try:
        get_trading_data(asset='BTC', start_date='2015-01-01', end_date=None)
    except Exception as e:
        msg = traceback.format_exc()
        logger.info(msg)
        send_error_to_email(script_name=os.path.basename(__file__), func_name="get_trading_data", message=msg)
        raise e
    task_to_db(os.path.basename(__file__), 'get_trading_data', 1)
    logger.info("成功get_trading_data, asset=BTC")

    # 全市场收益率分析
    logger.info("开始all_ret_analysis")
    task_to_db(os.path.basename(__file__), 'all_ret_analysis')
    try:
        all_ret_analysis()
    except Exception as e:
        msg = traceback.format_exc()
        logger.info(msg)
        send_error_to_email(script_name=os.path.basename(__file__), func_name="all_ret_analysis", message=msg)
        raise e
    task_to_db(os.path.basename(__file__), 'all_ret_analysis', 1)
    logger.info("成功all_ret_analysis")